Merge branch 'main' into fix/solr-10-migration

Author: dpol1

README.md

@@ -9,7 +9,7 @@ Apache StormCrawler is an open source collection of resources for building low-l
 
 ## Quickstart
 
-NOTE: These instructions assume that you have [Apache Maven](https://maven.apache.org/install.html) installed. You will need to install [Apache Storm 2.8.4](http://storm.apache.org/) to run the crawler.
+NOTE: These instructions assume that you have [Apache Maven](https://maven.apache.org/install.html) installed. You will need to install [Apache Storm 2.8.5](http://storm.apache.org/) to run the crawler.
 
 StormCrawler requires Java 17 or above. To execute tests, it requires you to have a locally installed and working Docker environment.
 

THIRD-PARTY.txt

@@ -5,7 +5,7 @@ Have a look at the code and resources and modify them to your heart's content.
 
 ## Native
 
-You need to install Apache Storm. The instructions on [setting up a Storm cluster](https://storm.apache.org/releases/2.8.4/Setting-up-a-Storm-cluster.html) should help.
+You need to install Apache Storm. The instructions on [setting up a Storm cluster](https://storm.apache.org/releases/2.8.5/Setting-up-a-Storm-cluster.html) should help.
 You also need to have an instance of URLFrontier running. See [the URLFrontier README](https://github.com/crawler-commons/url-frontier/tree/master/service); the easiest way is to use Docker, like so:
 
 ```

archetype/src/main/resources/archetype-resources/README.md

@@ -32,7 +32,7 @@ under the License.
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <stormcrawler.version>${project.version}</stormcrawler.version>
-        <storm.version>2.8.4</storm.version>
+        <storm.version>2.8.5</storm.version>
         <urlfrontier.version>2.4</urlfrontier.version>
     </properties>
 

archetype/src/main/resources/archetype-resources/pom.xml

@@ -39,7 +39,7 @@ under the License.
         <!-- dependency versions -->
         <crawler-commons.version>1.6</crawler-commons.version>
         <jsoup.version>1.22.1</jsoup.version>
-        <icu4j.version>78.2</icu4j.version>
+        <icu4j.version>78.3</icu4j.version>
         <xerces.version>2.12.2</xerces.version>
         <httpclient.version>4.5.14</httpclient.version>
         <snakeyaml.version>2.6</snakeyaml.version>
@@ -50,7 +50,6 @@ under the License.
         <okhttp.version>5.3.2</okhttp.version>
         <caffeine.version>3.2.3</caffeine.version>
         <xsoup.version>0.3.7</xsoup.version>
-        <awaitility.version>4.3.0</awaitility.version>
         <guava.version>33.5.0-jre</guava.version>
         <jacoco.haltOnFailure>true</jacoco.haltOnFailure>
         <jacoco.classRatio>0.72</jacoco.classRatio>
@@ -145,7 +144,6 @@ under the License.
         <dependency>
             <groupId>org.awaitility</groupId>
             <artifactId>awaitility</artifactId>
-            <version>${awaitility.version}</version>
             <scope>test</scope>
         </dependency>
 
@@ -268,7 +266,7 @@ under the License.
             <dependency>
                 <groupId>com.fasterxml.jackson.core</groupId>
                 <artifactId>jackson-annotations</artifactId>
-                <version>${jackson.version}</version>
+                <version>${jackson-annotations.version}</version>
             </dependency>
         </dependencies>
     </dependencyManagement>

core/pom.xml

@@ -52,6 +52,7 @@
 import org.apache.stormcrawler.persistence.Status;
 import org.apache.stormcrawler.protocol.ProtocolResponse;
 import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.URLUtil;
 import org.slf4j.LoggerFactory;
 import org.xml.sax.InputSource;
 
@@ -175,7 +176,7 @@ private List<Outlink> parseFeed(String url, byte[] content, Metadata parentMetad
             feed = input.build(new InputSource(is));
         }
 
-        URL url1 = new URL(url);
+        URL url1 = URLUtil.toURL(url);
 
         List<SyndEntry> entries = feed.getEntries();
         for (SyndEntry entry : entries) {

core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java

@@ -61,6 +61,7 @@
 import org.apache.stormcrawler.protocol.RobotRules;
 import org.apache.stormcrawler.util.ConfUtils;
 import org.apache.stormcrawler.util.PerSecondReducer;
+import org.apache.stormcrawler.util.URLUtil;
 import org.slf4j.LoggerFactory;
 
 /**
@@ -529,7 +530,7 @@ public void run() {
                 boolean asap = false;
 
                 try {
-                    URL url = new URL(fit.url);
+                    URL url = URLUtil.toURL(fit.url);
                     Protocol protocol = protocolFactory.getProtocol(url);
 
                     if (protocol == null) {
@@ -982,7 +983,7 @@ public void execute(Tuple input) {
         URL url;
 
         try {
-            url = new URL(urlString);
+            url = URLUtil.toURL(urlString);
         } catch (MalformedURLException e) {
             LOG.error("{} is a malformed URL", urlString);
 

core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java

@@ -292,7 +292,7 @@ public void execute(Tuple tuple) {
             } else {
                 final Elements links = jsoupDoc.select("a[href]");
                 slinks = new HashMap<>(links.size());
-                final URL baseUrl = new URL(url);
+                final URL baseUrl = URLUtil.toURL(url);
                 for (Element link : links) {
                     // nofollow
                     String[] relkeywords = link.attr("rel").split(" ");
@@ -374,7 +374,7 @@ public void execute(Tuple tuple) {
 
                     // https://github.com/apache/stormcrawler/issues/954
                     if (allowRedirs() && StringUtils.isNotBlank(redirection)) {
-                        emitOutlink(tuple, new URL(url), redirection, metadata);
+                        emitOutlink(tuple, URLUtil.toURL(url), redirection, metadata);
                     }
 
                     // Mark URL as redirected
@@ -515,7 +515,7 @@ protected List<Outlink> toOutlinks(
 
         URL sourceUrl;
         try {
-            sourceUrl = new URL(url);
+            sourceUrl = URLUtil.toURL(url);
         } catch (MalformedURLException e) {
             // we would have known by now as previous components check whether
             // the URL is valid

core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java

@@ -53,6 +53,7 @@
 import org.apache.stormcrawler.protocol.RobotRules;
 import org.apache.stormcrawler.util.ConfUtils;
 import org.apache.stormcrawler.util.PerSecondReducer;
+import org.apache.stormcrawler.util.URLUtil;
 import org.slf4j.LoggerFactory;
 
 /**
@@ -265,7 +266,7 @@ public void execute(Tuple input) {
         URL url;
 
         try {
-            url = new URL(urlString);
+            url = URLUtil.toURL(urlString);
         } catch (MalformedURLException e) {
             LOG.error("{} is a malformed URL", urlString);
             // Report to status stream and ack

core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java

@@ -61,6 +61,7 @@
 import org.apache.stormcrawler.persistence.DefaultScheduler;
 import org.apache.stormcrawler.persistence.Status;
 import org.apache.stormcrawler.util.ConfUtils;
+import org.apache.stormcrawler.util.URLUtil;
 import org.slf4j.LoggerFactory;
 
 /**
@@ -183,7 +184,7 @@ private List<Outlink> parseSiteMap(
             String url, byte[] content, String contentType, Metadata parentMetadata)
             throws UnknownFormatException, IOException {
 
-        URL url1 = new URL(url);
+        URL url1 = URLUtil.toURL(url);
         long start = System.currentTimeMillis();
         AbstractSiteMap siteMap;
         // let the parser guess what the mimetype is