NUTCH-2695: fix some alerts raised by LGTM

- useless checks for null in OutlinkExtractor, protocol-http, ParsePluginsReader, parse-html, parse-tika - called LOG.equals(...) instead of LOG.error(...) in ArcRecordReader - constant loop condition in protocol-http - implicit conversion from array to string in CrawlDbReducer - `Missing format argument` in log methods - spurious Javadoc @param tags
petapro · Feb 22, 2019 · 31ecf64 · 31ecf64
1 parent 3abe7db
commit 31ecf64
Show file tree

Hide file tree

Showing 20 changed files with 84 additions and 102 deletions.
diff --git a/src/java/org/apache/nutch/crawl/CrawlDbReducer.java b/src/java/org/apache/nutch/crawl/CrawlDbReducer.java
@@ -34,6 +34,7 @@
 import org.apache.nutch.metadata.Nutch;
 import org.apache.nutch.scoring.ScoringFilterException;
 import org.apache.nutch.scoring.ScoringFilters;
+import org.apache.nutch.util.StringUtil;
 
 /** Merge new page entries with existing entries. */
 public class CrawlDbReducer extends
@@ -169,7 +170,8 @@ public void reduce(Text key, Iterable<CrawlDatum> values,
         context.getCounter("CrawlDB status",
             CrawlDatum.getStatusName(old.getStatus())).increment(1);
       } else {
-        LOG.warn("Missing fetch and old value, signature=" + signature);
+        LOG.warn("Missing fetch and old value, signature="
+            + StringUtil.toHexString(signature));
       }
       return;
     }

diff --git a/src/java/org/apache/nutch/hostdb/UpdateHostDbMapper.java b/src/java/org/apache/nutch/hostdb/UpdateHostDbMapper.java
@@ -58,9 +58,6 @@ public class UpdateHostDbMapper
   protected URLFilters filters = null;
   protected URLNormalizers normalizers = null;
 
-  /**
-   * @param job
-   */
   @Override
   public void setup(Mapper<Text, Writable, Text, NutchWritable>.Context context) {
     Configuration conf = context.getConfiguration();

diff --git a/src/java/org/apache/nutch/hostdb/UpdateHostDbReducer.java b/src/java/org/apache/nutch/hostdb/UpdateHostDbReducer.java
@@ -71,8 +71,6 @@ public class UpdateHostDbReducer
 
   /**
     * Configures the thread pool and prestarts all resolver threads.
-    *
-    * @param job
     */
   @Override
   public void setup(Reducer<Text, NutchWritable, Text, HostDatum>.Context context) {

diff --git a/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java b/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
@@ -268,7 +268,7 @@ protected int process(String url, StringBuilder output) throws Exception {
 
     output.append("\n"); // For readability if keepClientCnxOpen
 
-    if (getConf().getBoolean("doIndex", false) && doc != null) {
+    if (getConf().getBoolean("doIndex", false)) {
       IndexWriters writers = IndexWriters.get(getConf());
       writers.open(getConf(), "IndexingFilterChecker");
       writers.write(doc);

diff --git a/src/java/org/apache/nutch/parse/OutlinkExtractor.java b/src/java/org/apache/nutch/parse/OutlinkExtractor.java
@@ -129,7 +129,7 @@ public static Outlink[] getOutlinks(final String plainText, String anchor,
     final Outlink[] retval;
 
     // create array of the Outlinks
-    if (outlinks != null && outlinks.size() > 0) {
+    if (outlinks.size() > 0) {
       retval = outlinks.toArray(new Outlink[0]);
     } else {
       retval = new Outlink[0];

diff --git a/src/java/org/apache/nutch/parse/ParseData.java b/src/java/org/apache/nutch/parse/ParseData.java
@@ -150,20 +150,10 @@ public final void readFields(DataInput in) throws IOException {
       outlinks[i] = Outlink.read(in);
     }
 
-    if (version < 3) {
-      int propertyCount = in.readInt(); // read metadata
-      contentMeta.clear();
-      for (int i = 0; i < propertyCount; i++) {
-        contentMeta.add(Text.readString(in), Text.readString(in));
-      }
-    } else {
-      contentMeta.clear();
-      contentMeta.readFields(in);
-    }
-    if (version > 3) {
-      parseMeta.clear();
-      parseMeta.readFields(in);
-    }
+    contentMeta.clear();
+    contentMeta.readFields(in);
+    parseMeta.clear();
+    parseMeta.readFields(in);
   }
 
   public final void write(DataOutput out) throws IOException {

diff --git a/src/java/org/apache/nutch/parse/ParsePluginsReader.java b/src/java/org/apache/nutch/parse/ParsePluginsReader.java
@@ -238,7 +238,7 @@ private Map<String, String> getAliases(Element parsePluginsRoot) {
     Map<String, String> aliases = new HashMap<>();
     NodeList aliasRoot = parsePluginsRoot.getElementsByTagName("aliases");
 
-    if (aliasRoot == null || (aliasRoot != null && aliasRoot.getLength() == 0)) {
+    if (aliasRoot == null || aliasRoot.getLength() == 0) {
       if (LOG.isWarnEnabled()) {
         LOG.warn("No aliases defined in parse-plugins.xml!");
       }

diff --git a/src/java/org/apache/nutch/segment/SegmentMerger.java b/src/java/org/apache/nutch/segment/SegmentMerger.java
@@ -392,15 +392,15 @@ public void map(Text key, MetaWrapper value,
         try {
           url = normalizers.normalize(url, URLNormalizers.SCOPE_DEFAULT); // normalize the url.
         } catch (Exception e) {
-          LOG.warn("Skipping {} :", url, e.getMessage());
+          LOG.warn("Skipping {} : {}", url, e.getMessage());
           url = null;
         }
       }
       if (url != null && filters != null) {
         try {
           url = filters.filter(url);
         } catch (Exception e) {
-          LOG.warn("Skipping key {} : ", url, e.getMessage());
+          LOG.warn("Skipping key {} : {}", url, e.getMessage());
           url = null;
         }
       }

diff --git a/src/java/org/apache/nutch/service/impl/LinkReader.java b/src/java/org/apache/nutch/service/impl/LinkReader.java
@@ -61,7 +61,7 @@ public List read(String path) throws FileNotFoundException {
 
     }catch (IOException e) {
       e.printStackTrace();
-      LOG.error("Error occurred while reading file {} : ", file, StringUtils.stringifyException(e));
+      LOG.error("Error occurred while reading file {} : {}", file, StringUtils.stringifyException(e));
       throw new WebApplicationException();
     } 
 
@@ -93,7 +93,7 @@ public List head(String path, int nrows) throws FileNotFoundException {
 
     }catch (IOException e) {
       e.printStackTrace();
-      LOG.error("Error occurred while reading file {} : ", file, StringUtils.stringifyException(e));
+      LOG.error("Error occurred while reading file {} : {}", file, StringUtils.stringifyException(e));
       throw new WebApplicationException();
     } 
 
@@ -127,7 +127,7 @@ public List slice(String path, int start, int end)
 
     }catch (IOException e) {
       e.printStackTrace();
-      LOG.error("Error occurred while reading file {} : ", file, StringUtils.stringifyException(e));
+      LOG.error("Error occurred while reading file {} : {}", file, StringUtils.stringifyException(e));
       throw new WebApplicationException();
     } 
 
@@ -151,7 +151,7 @@ public int count(String path) throws FileNotFoundException {
     } catch(FileNotFoundException fne){ 
       throw new FileNotFoundException();
     }catch (IOException e) {
-      LOG.error("Error occurred while reading file {} : ", file, StringUtils.stringifyException(e));
+      LOG.error("Error occurred while reading file {} : {}", file, StringUtils.stringifyException(e));
       throw new WebApplicationException();
     } 
     return i;

diff --git a/src/java/org/apache/nutch/service/impl/NodeReader.java b/src/java/org/apache/nutch/service/impl/NodeReader.java
@@ -61,7 +61,7 @@ public List read(String path) throws FileNotFoundException {
 
     }catch (IOException e) {
       e.printStackTrace();
-      LOG.error("Error occurred while reading file {} : ", file, StringUtils.stringifyException(e));
+      LOG.error("Error occurred while reading file {} : {}", file, StringUtils.stringifyException(e));
       throw new WebApplicationException();
     } 
 
@@ -93,7 +93,7 @@ public List head(String path, int nrows) throws FileNotFoundException {
 
     }catch (IOException e) {
       e.printStackTrace();
-      LOG.error("Error occurred while reading file {} : ", file, 
+      LOG.error("Error occurred while reading file {} : {}", file, 
           StringUtils.stringifyException(e));
       throw new WebApplicationException();
     } 
@@ -128,7 +128,7 @@ public List slice(String path, int start, int end)
 
     }catch (IOException e) {
       e.printStackTrace();
-      LOG.error("Error occurred while reading file {} : ", file, 
+      LOG.error("Error occurred while reading file {} : {}", file, 
           StringUtils.stringifyException(e));
       throw new WebApplicationException();
     } 
@@ -157,7 +157,7 @@ public int count(String path) throws FileNotFoundException {
 
     }catch (IOException e) {
       e.printStackTrace();
-      LOG.error("Error occurred while reading file {} : ", file, 
+      LOG.error("Error occurred while reading file {} : {}", file, 
           StringUtils.stringifyException(e));
       throw new WebApplicationException();
     } 

diff --git a/src/java/org/apache/nutch/service/impl/SequenceReader.java b/src/java/org/apache/nutch/service/impl/SequenceReader.java
@@ -64,7 +64,7 @@ public List<List<String>> read(String path) throws FileNotFoundException {
     }catch (IOException e) {
       // TODO Auto-generated catch block
       e.printStackTrace();
-      LOG.error("Error occurred while reading file {} : ", file, 
+      LOG.error("Error occurred while reading file {} : {}", file,
           StringUtils.stringifyException(e));
       throw new WebApplicationException();
     } 
@@ -99,7 +99,7 @@ public List<List<String>> head(String path, int nrows)
       throw new FileNotFoundException();
     }catch (IOException e) {
       // TODO Auto-generated catch block
-      LOG.error("Error occurred while reading file {} : ", file, 
+      LOG.error("Error occurred while reading file {} : {}", file,
           StringUtils.stringifyException(e));
       throw new WebApplicationException();
     } 
@@ -134,7 +134,7 @@ public List<List<String>> slice(String path, int start, int end)
       throw new FileNotFoundException();
     }catch (IOException e) {
       // TODO Auto-generated catch block
-      LOG.error("Error occurred while reading file {} : ", file, 
+      LOG.error("Error occurred while reading file {} : {}", file,
           StringUtils.stringifyException(e));
       throw new WebApplicationException();
     } 
@@ -161,7 +161,7 @@ public int count(String path) throws FileNotFoundException {
       throw new FileNotFoundException();
     }catch (IOException e) {
       // TODO Auto-generated catch block
-      LOG.error("Error occurred while reading file {} : ", file, 
+      LOG.error("Error occurred while reading file {} : {}", file,
           StringUtils.stringifyException(e));
       throw new WebApplicationException();
     } 

diff --git a/src/java/org/apache/nutch/tools/arc/ArcInputFormat.java b/src/java/org/apache/nutch/tools/arc/ArcInputFormat.java
@@ -46,8 +46,8 @@ public RecordReader<Text, BytesWritable> createRecordReader(InputSplit split,
    *          The InputSplit of the arc file to process.
    * @param job
    *          The job configuration.
-   * @param reporter
-   *          The progress reporter.
+   * @param context
+   *          The task context.
    */
   public RecordReader<Text, BytesWritable> getRecordReader(InputSplit split,
       Job job, Context context) throws IOException {

diff --git a/src/java/org/apache/nutch/tools/arc/ArcRecordReader.java b/src/java/org/apache/nutch/tools/arc/ArcRecordReader.java
@@ -308,7 +308,7 @@ public boolean next(Text key, BytesWritable value) throws IOException {
         return true;
       }
     } catch (Exception e) {
-      LOG.equals(StringUtils.stringifyException(e));
+      LOG.error("Failed reading ARC record: ", e);
     }
 
     // couldn't populate the record or there is no next record to read

diff --git a/src/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java b/src/java/org/apache/nutch/tools/arc/ArcSegmentCreator.java
@@ -269,8 +269,8 @@ private ParseStatus output(Context context,
      * and other relevant data.
      * </p>
      * 
-     * @param job
-     *          The job configuration.
+     * @param context
+     *          The task context.
      */
     public void setup(Mapper<Text, BytesWritable, Text, NutchWritable>.Context context) { 
       // set the url filters, scoring filters the parse util and the url

diff --git a/src/java/org/apache/nutch/util/EncodingDetector.java b/src/java/org/apache/nutch/util/EncodingDetector.java
@@ -170,10 +170,8 @@ public void autoDetectClues(Content content, boolean filter) {
       // will sometimes throw exceptions
       try {
         detector.enableInputFilter(filter);
-        if (data.length > MIN_LENGTH) {
-          detector.setText(data);
-          matches = detector.detectAll();
-        }
+        detector.setText(data);
+        matches = detector.detectAll();
       } catch (Exception e) {
         LOG.debug("Exception from ICU4J (ignoring): ", e);
       }

diff --git a/src/java/org/apache/nutch/util/MimeUtil.java b/src/java/org/apache/nutch/util/MimeUtil.java
@@ -166,8 +166,7 @@ public String autoResolveContentType(String typeName, String url, byte[] data) {
     }
 
     // if returned null, or if it's the default type then try url resolution
-    if (type == null
-        || (type != null && type.getName().equals(MimeTypes.OCTET_STREAM))) {
+    if (type == null || type.getName().equals(MimeTypes.OCTET_STREAM)) {
       // If no mime-type header, or cannot find a corresponding registered
       // mime-type, then guess a mime-type from the url pattern
       try {

diff --git a/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HTMLMetaProcessor.java b/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HTMLMetaProcessor.java
@@ -81,35 +81,32 @@ private static final void getMetaTagsHelper(HTMLMetaTags metaTags, Node node,
             String name = nameNode.getNodeValue().toLowerCase();
             metaTags.getGeneralTags().add(name, contentNode.getNodeValue());
             if ("robots".equals(name)) {
+              String directives = contentNode.getNodeValue().toLowerCase();
+              int index = directives.indexOf("none");
 
-              if (contentNode != null) {
-                String directives = contentNode.getNodeValue().toLowerCase();
-                int index = directives.indexOf("none");
-
-                if (index >= 0) {
-                  metaTags.setNoIndex();
-                  metaTags.setNoFollow();
-                }
+              if (index >= 0) {
+                metaTags.setNoIndex();
+                metaTags.setNoFollow();
+              }
 
-                index = directives.indexOf("all");
-                if (index >= 0) {
-                  // do nothing...
-                }
+              index = directives.indexOf("all");
+              if (index >= 0) {
+                // do nothing...
+              }
 
-                index = directives.indexOf("noindex");
-                if (index >= 0) {
-                  metaTags.setNoIndex();
-                }
+              index = directives.indexOf("noindex");
+              if (index >= 0) {
+                metaTags.setNoIndex();
+              }
 
-                index = directives.indexOf("nofollow");
-                if (index >= 0) {
-                  metaTags.setNoFollow();
-                }
+              index = directives.indexOf("nofollow");
+              if (index >= 0) {
+                metaTags.setNoFollow();
+              }
 
-                index = directives.indexOf("noarchive");
-                if (index >= 0) {
-                  metaTags.setNoCache();
-                }
+              index = directives.indexOf("noarchive");
+              if (index >= 0) {
+                metaTags.setNoCache();
               }
 
             } // end if (name == robots)

diff --git a/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/HTMLMetaProcessor.java b/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/HTMLMetaProcessor.java
@@ -82,35 +82,32 @@ private static final void getMetaTagsHelper(HTMLMetaTags metaTags, Node node,
             String name = nameNode.getNodeValue().toLowerCase();
             metaTags.getGeneralTags().add(name, contentNode.getNodeValue());
             if ("robots".equals(name)) {
+              String directives = contentNode.getNodeValue().toLowerCase();
+              int index = directives.indexOf("none");
 
-              if (contentNode != null) {
-                String directives = contentNode.getNodeValue().toLowerCase();
-                int index = directives.indexOf("none");
-
-                if (index >= 0) {
-                  metaTags.setNoIndex();
-                  metaTags.setNoFollow();
-                }
+              if (index >= 0) {
+                metaTags.setNoIndex();
+                metaTags.setNoFollow();
+              }
 
-                index = directives.indexOf("all");
-                if (index >= 0) {
-                  // do nothing...
-                }
+              index = directives.indexOf("all");
+              if (index >= 0) {
+                // do nothing...
+              }
 
-                index = directives.indexOf("noindex");
-                if (index >= 0) {
-                  metaTags.setNoIndex();
-                }
+              index = directives.indexOf("noindex");
+              if (index >= 0) {
+                metaTags.setNoIndex();
+              }
 
-                index = directives.indexOf("nofollow");
-                if (index >= 0) {
-                  metaTags.setNoFollow();
-                }
+              index = directives.indexOf("nofollow");
+              if (index >= 0) {
+                metaTags.setNoFollow();
+              }
 
-                index = directives.indexOf("noarchive");
-                if (index >= 0) {
-                  metaTags.setNoCache();
-                }
+              index = directives.indexOf("noarchive");
+              if (index >= 0) {
+                metaTags.setNoCache();
               }
 
             } // end if (name == robots)