Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HDDS-11591. Copy dependencies when building each module #7325

Merged
merged 1 commit into from
Oct 29, 2024
Merged

HDDS-11591. Copy dependencies when building each module #7325

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 0 additions & 31 deletions hadoop-hdds/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -322,36 +322,5 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
</plugins>
</build>
</profile>
<profile>
<id>add-classpath-descriptor</id>
<activation>
<file>
<exists>src/main/java</exists>
</file>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>add-classpath-descriptor</id>
<phase>prepare-package</phase>
<goals>
<goal>build-classpath</goal>
</goals>
<configuration>
<outputFile>${project.build.outputDirectory}/${project.artifactId}.classpath</outputFile>
<prefix>$HDDS_LIB_JARS_DIR</prefix>
<outputFilterFile>true</outputFilterFile>
<includeScope>runtime</includeScope>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
2 changes: 1 addition & 1 deletion hadoop-hdds/rocks-native/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,7 @@
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-jars</id>
<id>copy-dependencies</id>
<phase>process-sources</phase>
<goals>
<goal>copy-dependencies</goal>
Expand Down
1 change: 1 addition & 0 deletions hadoop-hdds/server-scm/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<packaging>jar</packaging>

<properties>
<classpath.skip>false</classpath.skip>
</properties>

<dependencies>
Expand Down
1 change: 1 addition & 0 deletions hadoop-ozone/csi/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<packaging>jar</packaging>

<properties>
<classpath.skip>false</classpath.skip>
<maven.test.skip>true</maven.test.skip> <!-- no tests in this module so far -->
</properties>

Expand Down
1 change: 1 addition & 0 deletions hadoop-ozone/datanode/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
<version>2.0.0-SNAPSHOT</version>

<properties>
<classpath.skip>false</classpath.skip>
<maven.test.skip>true</maven.test.skip> <!-- no tests in this module so far -->
<spotbugs.skip>true</spotbugs.skip>
</properties>
Expand Down
4 changes: 2 additions & 2 deletions hadoop-ozone/dev-support/checks/dependency.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ cp ${src_dir}/current.txt "$REPORT_DIR"/

#implementation of sort cli is not exactly the same everywhere. It's better to sort with the same command locally
(diff -uw \
<(sort ${src_dir}/jar-report.txt) \
<(sort ${src_dir}/current.txt) \
<(sort -u ${src_dir}/jar-report.txt) \
<(sort -u ${src_dir}/current.txt) \
|| true) \
> "$REPORT_FILE"

Expand Down
10 changes: 10 additions & 0 deletions hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ run cp -p "${ROOT}/HISTORY.md" .
run cp -p "${ROOT}/SECURITY.md" .
run cp -p "${ROOT}/CONTRIBUTING.md" .

run mkdir -p ./share/ozone/classpath
run mkdir -p ./share/ozone/lib
run mkdir -p ./share/ozone/web
run mkdir -p ./bin
run mkdir -p ./sbin
Expand Down Expand Up @@ -129,6 +131,14 @@ run cp -p -r "${ROOT}/hadoop-ozone/dist/target/Dockerfile" .
#Copy pre-generated keytabs
run cp -p -R "${ROOT}/hadoop-ozone/dist/src/main/keytabs" compose/_keytabs

for file in $(find "${ROOT}" -path '*/target/classes/*.classpath' | sort); do
cp -n -p -v "$file" share/ozone/classpath/
done

for file in $(find "${ROOT}" -path '*/share/ozone/lib/*jar' | sort); do
cp -n -p -v "$file" share/ozone/lib/
done

#workaround for https://issues.apache.org/jira/browse/MRESOURCES-236
find ./compose -name "*.sh" -exec chmod 755 {} \;
find ./kubernetes -name "*.sh" -exec chmod 755 {} \;
37 changes: 1 addition & 36 deletions hadoop-ozone/dist/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,24 +71,7 @@
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-classpath-files</id>
<phase>prepare-package</phase>
<goals>
<goal>unpack-dependencies</goal>
</goals>
<configuration>
<outputDirectory>
target/ozone-${ozone.version}/share/ozone/classpath
</outputDirectory>
<includes>*.classpath</includes>
<includeArtifactIds>
hdds-server-scm,ozone-common,ozone-csi,ozone-datanode,ozone-httpfsgateway,
ozone-insight,ozone-manager,ozone-recon,ozone-s3gateway,ozone-tools,hdds-rocks-native,ozone-s3-secret-store
</includeArtifactIds>
</configuration>
</execution>
<execution>
<id>copy-jars</id>
<id>copy-omitted-jars</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-dependencies</goal>
Expand All @@ -98,24 +81,6 @@
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-omitted-jars</id>
<phase>prepare-package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>target/ozone-${ozone.version}/share/ozone/lib
</outputDirectory>
<artifactItems>
<artifactItem>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${grpc.protobuf-compile.version}</version>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
Expand Down
1 change: 1 addition & 0 deletions hadoop-ozone/httpfsgateway/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
<description>Apache Ozone HttpFS</description>

<properties>
<classpath.skip>false</classpath.skip>
<httpfs.source.repository>REPO NOT AVAIL</httpfs.source.repository>
<httpfs.source.revision>REVISION NOT AVAIL</httpfs.source.revision>
<maven.build.timestamp.format>yyyy-MM-dd'T'HH:mm:ssZ</maven.build.timestamp.format>
Expand Down
2 changes: 2 additions & 0 deletions hadoop-ozone/insight/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<description>Apache Ozone Insight Tool</description>
<name>Apache Ozone Insight Tool</name>
<packaging>jar</packaging>

<properties>
<classpath.skip>false</classpath.skip>
</properties>

<dependencies>
Expand Down
1 change: 1 addition & 0 deletions hadoop-ozone/ozone-manager/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<packaging>jar</packaging>

<properties>
<classpath.skip>false</classpath.skip>
</properties>

<dependencies>
Expand Down
31 changes: 0 additions & 31 deletions hadoop-ozone/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -455,36 +455,5 @@
</plugins>
</build>
</profile>
<profile>
<id>add-classpath-descriptor</id>
<activation>
<file>
<exists>src/main/java</exists>
</file>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>add-classpath-descriptor</id>
<phase>prepare-package</phase>
<goals>
<goal>build-classpath</goal>
</goals>
<configuration>
<outputFile>${project.build.outputDirectory}/${project.artifactId}.classpath</outputFile>
<prefix>$HDDS_LIB_JARS_DIR</prefix>
<outputFilterFile>true</outputFilterFile>
<includeScope>runtime</includeScope>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
1 change: 1 addition & 0 deletions hadoop-ozone/recon/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>ozone-recon</artifactId>
<properties>
<classpath.skip>false</classpath.skip>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Correct me if I am wrong here, I think the classpath.skip should be set to false for ozone-common module as well, isn't it? I am not able to find the file being updated here.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any particular reason why we don't need the ozone-common classpath file?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are right in that the build used to copy the classpath file for ozone-common module to share/ozone/classpath:

ozone/hadoop-ozone/dist/pom.xml

Lines 80 to 85 in 91d41a0

<outputDirectory>
target/ozone-${ozone.version}/share/ozone/classpath
</outputDirectory>
<includes>*.classpath</includes>
<includeArtifactIds>
hdds-server-scm,ozone-common,ozone-csi,ozone-datanode,ozone-httpfsgateway,

With this PR, classpath file for ozone-common is no longer created. The reason for this is the following:

  • Classpath files are used for running Ozone commands.
  • The specific classpath file to be used is determined by setting OZONE_RUN_ARTIFACT_NAME in hadoop-ozone/dist/src/shell/ozone/ozone.
  • ozone-common is not used for any commands as OZONE_RUN_ARTIFACT_NAME.
  • Therefore the classpath file is unused.

The last (maybe only) command which had used ozone-common as OZONE_RUN_ARTIFACT_NAME was version, replaced by 8d3817c (HDDS-4758) with ozone-tools.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh ok, this makes sense

<pnpm.version>8.15.7</pnpm.version>
</properties>
<build>
Expand Down
1 change: 1 addition & 0 deletions hadoop-ozone/s3gateway/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
<packaging>jar</packaging>
<version>2.0.0-SNAPSHOT</version>
<properties>
<classpath.skip>false</classpath.skip>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
</properties>
Expand Down
4 changes: 4 additions & 0 deletions hadoop-ozone/tools/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Ozone Tools</name>
<packaging>jar</packaging>

<properties>
<classpath.skip>false</classpath.skip>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.ozone</groupId>
Expand Down
34 changes: 34 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

<classpath.skip>true</classpath.skip>

<maven-gpg-plugin.version>3.2.7</maven-gpg-plugin.version>
<shell-executable>bash</shell-executable>

Expand Down Expand Up @@ -1703,6 +1705,38 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
</pluginManagement>

<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>add-classpath-descriptor</id>
<phase>prepare-package</phase>
<goals>
<goal>build-classpath</goal>
</goals>
<configuration>
<outputFile>${project.build.outputDirectory}/${project.artifactId}.classpath</outputFile>
<prefix>$HDDS_LIB_JARS_DIR</prefix>
<outputFilterFile>true</outputFilterFile>
<includeScope>runtime</includeScope>
<skip>${classpath.skip}</skip>
</configuration>
</execution>
<execution>
<id>copy-jars</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/share/ozone/lib</outputDirectory>
<includeScope>runtime</includeScope>
<skip>${classpath.skip}</skip>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<configuration>
Expand Down