pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.Crawler</groupId>
    <artifactId>Crawler</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>16</maven.compiler.source>
        <maven.compiler.target>16</maven.compiler.target>
    </properties>
    <dependencies>

        <dependency>
            <groupId>com.panforge</groupId>
            <artifactId>robots</artifactId>
            <version>1.4.6</version>
        </dependency>
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.13.1</version>
        </dependency>

        <dependency>
            <groupId>com.panforge</groupId>
            <artifactId>robots-http</artifactId>
            <version>1.0.10</version>
        </dependency>

        <dependency>
            <groupId>com.shekhargulati.urlcleaner</groupId>
            <artifactId>urlcleaner</artifactId>
            <version>0.4.0</version>
            <type>jar</type>
        </dependency>


        <dependency>
            <groupId>org.mongodb</groupId>
            <artifactId>mongo-java-driver</artifactId>
            <version>3.12.11</version>
        </dependency>


    </dependencies>
        </project>