Merge branch 'InseeFr:develop' into develop

.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
           java-version: 17
           distribution: "adopt"
       - name: Test, package and analyze with maven & SonarCloud
-        run: mvn verify sonar:sonar -Dsonar.projectKey=InseeFr_Trevas -Dsonar.organization=inseefr -Dsonar.host.url=https://sonarcloud.io
+        run: mvn verify sonar:sonar -Pcoverage -Dsonar.projectKey=InseeFr_Trevas -Dsonar.organization=inseefr -Dsonar.host.url=https://sonarcloud.io
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

coverage/pom.xml

@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>fr.insee.trevas</groupId>
+        <artifactId>trevas-parent</artifactId>
+        <version>1.3.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>coverage</artifactId>
+    <description>Aggregate Coverage Report</description>
+
+    <properties>
+        <sonar.coverage.jacoco.xmlReportPaths>
+            ${project.basedir}/../coverage/target/site/jacoco-aggregate/jacoco.xml
+        </sonar.coverage.jacoco.xmlReportPaths>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>fr.insee.trevas</groupId>
+            <artifactId>vtl-engine</artifactId>
+            <version>1.3.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>fr.insee.trevas</groupId>
+            <artifactId>vtl-jackson</artifactId>
+            <version>1.3.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>fr.insee.trevas</groupId>
+            <artifactId>vtl-jdbc</artifactId>
+            <version>1.3.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>fr.insee.trevas</groupId>
+            <artifactId>vtl-model</artifactId>
+            <version>1.3.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>fr.insee.trevas</groupId>
+            <artifactId>vtl-parser</artifactId>
+            <version>1.3.0-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>fr.insee.trevas</groupId>
+            <artifactId>vtl-spark</artifactId>
+            <version>1.3.0-SNAPSHOT</version>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.jacoco</groupId>
+                <artifactId>jacoco-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>report-aggregate</id>
+                        <phase>verify</phase>
+                        <goals>
+                            <goal>report-aggregate</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

docs/yarn.lock

@@ -4692,9 +4692,9 @@ flux@^4.0.1:
     fbjs "^3.0.1"
 
 follow-redirects@^1.0.0, follow-redirects@^1.14.7:
-  version "1.15.2"
-  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
-  integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
+  version "1.15.4"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.4.tgz#cdc7d308bf6493126b17ea2191ea0ccf3e535adf"
+  integrity sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==
 
 for-each@^0.3.3:
   version "0.3.3"

pom.xml

@@ -13,6 +13,7 @@
         <module>vtl-jackson</module>
         <module>vtl-spark</module>
         <module>vtl-jdbc</module>
+        <module>coverage</module>
     </modules>
 
     <name>Trevas</name>
@@ -68,6 +69,9 @@
         <jdk.version>11</jdk.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <jacoco.version>0.8.7</jacoco.version>
+        <sonar.coverage.jacoco.xmlReportPaths>
+            ${project.basedir}/coverage/target/site/jacoco-aggregate/jacoco.xml
+        </sonar.coverage.jacoco.xmlReportPaths>
     </properties>
 
     <dependencies>
@@ -151,6 +155,11 @@
                         <goals>
                             <goal>report</goal>
                         </goals>
+                        <configuration>
+                            <formats>
+                                <format>XML</format>
+                            </formats>
+                        </configuration>
                     </execution>
                 </executions>
             </plugin>
@@ -279,6 +288,46 @@
                 </plugins>
             </reporting>
         </profile>
+        <profile>
+            <id>coverage</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.jacoco</groupId>
+                        <artifactId>jacoco-maven-plugin</artifactId>
+                        <version>${jacoco.version}</version>
+                        <executions>
+                            <!-- Prepare the property pointing to the JaCoCo runtime agent which is
+                                 passed as VM argument when the Maven Surefire plugin is executed. -->
+                            <execution>
+                                <id>pre-unit-test</id>
+                                <goals>
+                                    <goal>prepare-agent</goal>
+                                </goals>
+                                <configuration>
+                                    <!-- Set the name of the property containing the settings for JaCoCo
+                                         runtime agent. -->
+                                    <propertyName>surefireArgLine</propertyName>
+                                    <append>true</append>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>post-unit-test</id>
+                                <phase>test</phase>
+                                <goals>
+                                    <goal>report</goal>
+                                </goals>
+                                <configuration>
+                                    <formats>
+                                        <format>XML</format>
+                                    </formats>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
     </profiles>
 
     <developers>

vtl-engine/pom.xml

@@ -15,6 +15,12 @@
     <description>VTL engine framework for Trevas</description>
     <version>1.3.0-SNAPSHOT</version>
 
+    <properties>
+        <sonar.coverage.jacoco.xmlReportPaths>
+            ${project.basedir}/../coverage/target/site/jacoco-aggregate/jacoco.xml
+        </sonar.coverage.jacoco.xmlReportPaths>
+    </properties>
+
     <dependencies>
         <dependency>
             <groupId>fr.insee.trevas</groupId>

vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/AnalyticsVisitor.java

@@ -202,4 +202,15 @@ public DatasetExpression visitRatioToReportAn(VtlParser.RatioToReportAnContext c
                 toPartitionBy(ctx.partition)
         );
     }
+
+    @Override
+    public DatasetExpression visitRankAn(VtlParser.RankAnContext ctx) {
+        return processingEngine.executeRankAn(
+                dataset,
+                this.targetColumnName,
+                toFunctionEnum(ctx.op, ctx),
+                toPartitionBy(ctx.partition),
+                toOrderBy(ctx.orderBy)
+        );
+    }
 }

vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ExpressionVisitor.java

@@ -3,6 +3,7 @@
 import fr.insee.vtl.engine.VtlScriptEngine;
 import fr.insee.vtl.engine.exceptions.UnimplementedException;
 import fr.insee.vtl.engine.exceptions.VtlRuntimeException;
+import fr.insee.vtl.engine.visitors.AnalyticsVisitor;
 import fr.insee.vtl.engine.visitors.ClauseVisitor;
 import fr.insee.vtl.engine.visitors.expression.functions.ComparisonFunctionsVisitor;
 import fr.insee.vtl.engine.visitors.expression.functions.DistanceFunctionsVisitor;
@@ -24,9 +25,12 @@
 import fr.insee.vtl.parser.VtlParser;
 
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static fr.insee.vtl.engine.VtlScriptEngine.fromContext;
 
@@ -353,6 +357,135 @@ public ResolvableExpression visitClauseExpr(VtlParser.ClauseExprContext ctx) {
         return clauseVisitor.visit(ctx.clause);
     }
 
+    @Override
+    public ResolvableExpression visitRatioToReportAn(VtlParser.RatioToReportAnContext ctx) {
+        var datasetExpression = (DatasetExpression) visit(ctx.expr());
+        var dsName = ctx.expr().getText();
+        var structure = datasetExpression.getDataStructure();
+        var ids = structure.getIdentifiers();
+
+        Map<String, DatasetExpression> analytics = new LinkedHashMap<>();
+        for (Structured.Component measure : structure.getMeasures()) {
+
+            // Convert each measure to an equivalent [calc xx := analyticFunction].
+            // where xx is [function name]_[original_measure_name]
+            List<String> colNames = Stream.concat(
+                    ids.stream(),
+                    Stream.of(measure)
+            ).map(Structured.Component::getName).collect(Collectors.toList());
+
+            // Filter out measure we don't need and rename the measure to the name of
+            // the dataset.
+            var tempDs = processingEngine.executeProject(datasetExpression,
+                    colNames);
+            tempDs = processingEngine.executeRename(tempDs,
+                    Map.of(measure.getName(), dsName));
+
+            // Execute the calc.
+            String targetColumnName = ctx.op.getText() + "_" + measure.getName();
+            AnalyticsVisitor analyticsVisitor = new AnalyticsVisitor(processingEngine, tempDs,
+                    targetColumnName);
+            var result = analyticsVisitor.visit(ctx);
+
+            // Rename back to the original name.
+            result = processingEngine.executeRename(result, Map.of(targetColumnName, measure.getName()));
+
+            // Drop the ds1 column.
+            result = processingEngine.executeProject(result,
+                    result.getColumnNames().stream().filter(n -> !n.equals(dsName)).collect(Collectors.toList()));
+
+            analytics.put(targetColumnName, result);
+
+        }
+        return processingEngine.executeInnerJoin(analytics);
+    }
+
+    @Override
+    public ResolvableExpression visitLagOrLeadAn(VtlParser.LagOrLeadAnContext ctx) {
+        var datasetExpression = (DatasetExpression) visit(ctx.expr());
+        var dsName = ctx.expr().getText();
+        var structure = datasetExpression.getDataStructure();
+        var ids = structure.getIdentifiers();
+
+        Map<String, DatasetExpression> analytics = new LinkedHashMap<>();
+        for (Structured.Component measure : structure.getMeasures()) {
+
+            // Convert each measure to an equivalent [calc xx := analyticFunction].
+            // where xx is [function name]_[original_measure_name]
+            List<String> colNames = Stream.concat(
+                    ids.stream(),
+                    Stream.of(measure)
+            ).map(Structured.Component::getName).collect(Collectors.toList());
+
+            // Filter out measure we don't need and rename the measure to the name of
+            // the dataset.
+            var tempDs = processingEngine.executeProject(datasetExpression,
+                    colNames);
+            tempDs = processingEngine.executeRename(tempDs,
+                    Map.of(measure.getName(), dsName));
+
+            // Execute the calc.
+            String targetColumnName = ctx.op.getText() + "_" + measure.getName();
+            AnalyticsVisitor analyticsVisitor = new AnalyticsVisitor(processingEngine, tempDs,
+                    targetColumnName);
+            var result = analyticsVisitor.visit(ctx);
+
+            // Rename back to the original name.
+            result = processingEngine.executeRename(result, Map.of(targetColumnName, measure.getName()));
+
+            // Drop the ds1 column.
+            result = processingEngine.executeProject(result,
+                    result.getColumnNames().stream().filter(n -> !n.equals(dsName)).collect(Collectors.toList()));
+
+            analytics.put(targetColumnName, result);
+
+        }
+        return processingEngine.executeInnerJoin(analytics);
+    }
+
+    @Override
+    public DatasetExpression visitAnSimpleFunction(VtlParser.AnSimpleFunctionContext ctx) {
+        var datasetExpression = (DatasetExpression) visit(ctx.expr());
+        var dsName = ctx.expr().getText();
+        var structure = datasetExpression.getDataStructure();
+        var ids = structure.getIdentifiers();
+
+        Map<String, DatasetExpression> analytics = new LinkedHashMap<>();
+        for (Structured.Component measure : structure.getMeasures()) {
+
+            // Convert each measure to an equivalent [calc xx := analyticFunction].
+            // where xx is [function name]_[original_measure_name]
+            List<String> colNames = Stream.concat(
+                    ids.stream(),
+                    Stream.of(measure)
+            ).map(Structured.Component::getName).collect(Collectors.toList());
+
+            // Filter out measure we don't need and rename the measure to the name of
+            // the dataset.
+            var tempDs = processingEngine.executeProject(datasetExpression,
+                    colNames);
+            tempDs = processingEngine.executeRename(tempDs,
+                    Map.of(measure.getName(), dsName));
+
+            // Execute the calc.
+            String targetColumnName = ctx.op.getText() + "_" + measure.getName();
+            AnalyticsVisitor analyticsVisitor = new AnalyticsVisitor(processingEngine, tempDs,
+                    targetColumnName);
+            var result = analyticsVisitor.visit(ctx);
+
+            // Rename back to the original name.
+            result = processingEngine.executeRename(result, Map.of(targetColumnName, measure.getName()));
+
+            // Drop the ds1 column.
+            result = processingEngine.executeProject(result,
+                    result.getColumnNames().stream().filter(n -> !n.equals(dsName)).collect(Collectors.toList()));
+
+            analytics.put(targetColumnName, result);
+
+        }
+        return processingEngine.executeInnerJoin(analytics);
+    }
+
     @Override
     public ResolvableExpression visitFunctionsExpression(VtlParser.FunctionsExpressionContext ctx) {
         ResolvableExpression expr = super.visitFunctionsExpression(ctx);

vtl-engine/src/test/java/fr/insee/vtl/engine/exceptions/UnimplementedExceptionTest.java

@@ -31,6 +31,6 @@ public void testSimple() {
 
         assertThatThrownBy(() -> {
             engine.eval(vtlExpression);
-        }).isInstanceOf(UnimplementedException.class);
+        }).isInstanceOf(UnsupportedOperationException.class);
     }
 }

vtl-jackson/pom.xml

@@ -15,6 +15,12 @@
     <description>Jackson module for the VTL model</description>
     <version>1.3.0-SNAPSHOT</version>
 
+    <properties>
+        <sonar.coverage.jacoco.xmlReportPaths>
+            ${project.basedir}/../coverage/target/site/jacoco-aggregate/jacoco.xml
+        </sonar.coverage.jacoco.xmlReportPaths>
+    </properties>
+
     <dependencies>
         <dependency>
             <groupId>fr.insee.trevas</groupId>

vtl-jdbc/pom.xml

@@ -15,6 +15,12 @@
     <description>Dataset wrapper around JDBC result set. Useful to expose SQL tables as VTL datasets</description>
     <version>1.3.0-SNAPSHOT</version>
 
+    <properties>
+        <sonar.coverage.jacoco.xmlReportPaths>
+            ${project.basedir}/../coverage/target/site/jacoco-aggregate/jacoco.xml
+        </sonar.coverage.jacoco.xmlReportPaths>
+    </properties>
+
     <dependencies>
         <dependency>
             <groupId>fr.insee.trevas</groupId>