NUTCH-3197 Yetus: fix product codespell typos flagged by precommit

.codespellrc

@@ -3,9 +3,10 @@
 # Loction: intentional misspelling in protocol-http/okhttp TestResponse fixtures
 # used to exercise SpellCheckedMetadata error-tolerant Location header matching.
 # uery: intentional query-string token in urlnormalizer-protocol TestProtocolURLNormalizer.
-ignore-words-list = AfterAll,BeforeAll,Loction,uery
+# fo: intentional trie matcher test prefix in PrefixStringMatcher/SuffixStringMatcher demo mains.
+ignore-words-list = AfterAll,BeforeAll,Loction,uery,fo
 # CHANGES.md is a historical changelog not maintained by this branch.
 # Language-identifier corpora, binary license trees, and naivebayes training
 # sample are also listed in .yetus/excludes.txt for full-tree Yetus runs; skip
 # them here so local codespell matches CI.
-skip = CHANGES.md,src/plugin/language-identifier/src/test,LICENSE-binary,NOTICE-binary,licenses-binary,conf/naivebayes-train.txt.template
+skip = CHANGES.md,src/plugin/language-identifier/src/test,LICENSE-binary,NOTICE-binary,licenses-binary,conf/naivebayes-train.txt.template,src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/langmappings.properties,**/naivebayes-model,**/sample/**

.hadolint.yaml

@@ -0,0 +1,4 @@
+# Hadolint config for docker/Dockerfile (NUTCH-3197).
+# DL3018: do not pin floating Alpine apk versions without a deliberate bump policy.
+ignored:
+  - DL3018

.yetus/excludes.txt

@@ -14,3 +14,6 @@
 #
 # Naive Bayes training sample (deliberately messy email/OCR-like text).
 ^conf/naivebayes-train\.txt\.template$
+#
+# parse-js sample: test fixture for link extraction, not production JS
+^src/plugin/parse-js/sample/

README.md

@@ -48,7 +48,7 @@ test-patch --basedir=/path/to/clean/repo --build-tool=nobuild \
 ```
 
 Exclude patterns and related Yetus baselines can be added under `.yetus/`
-(see `.yetus/excludes.txt`).
+(see `.yetus/excludes.txt`, `.yetus/detsecrets-ignored-hashes.txt`).
 
 IDE setup
 ---------

conf/nutch-default.xml

@@ -62,7 +62,7 @@
   <description>If true, no file content will be saved during fetch.
   And it is probably what we want to set most of time, since file:// URLs
   are meant to be local and we can always use them directly at parsing
-  and indexing stages. Otherwise file contents will be saved.
+  and indexing stags. Otherwise file contents will be saved.
   !! NOT IMPLEMENTED YET !!
   </description>
 </property>
@@ -1361,7 +1361,7 @@
   <value>-1</value>
   <description>
     The maximum time in seconds fetcher will cache redirects for
-    deduplication.  If the same redirect URL is seen again withing
+    deduplication.  If the same redirect URL is seen again within
     this time it is skipped. This allows to avoid pathological cases
     where many or most of the URLs of a host are redirected to the
     same URL, eg. a page to login, accept cookies, indicating an
@@ -1453,7 +1453,7 @@
 <property>
   <name>indexer.score.power</name>
   <value>0.5</value>
-  <description>Determines the power of link analyis scores. The boost
+  <description>Determines the power of link analysis scores. The boost
   of each page is set to <i>score<sup>scorePower</sup></i> where
   <i>score</i> is its link analysis score and <i>scorePower</i> is the
   value of this parameter.  This is compiled into indexes, so, when
@@ -1745,7 +1745,7 @@
   <name>parser.html.line.separators</name>
   <value>article,aside,blockquote,canvas,dd,div,dl,dt,fieldset,figcaption,figure,footer,form,h1,h2,h3,h4,h5,h6,header,hr,li,main,nav,noscript,ol,output,p,pre,section,table,tfoot,ul,video</value>
  <description>Comma separated list of HTML tags. Newline will be added to the
-  parsed text after these tages.
+  parsed text after these tags.
   The default list above are the block-level HTML elements.
   Tags must be in lower case.
   To disable this feature, leave the list empty.</description>

docker/Dockerfile

@@ -30,26 +30,25 @@ LABEL org.opencontainers.image.vendor="Apache Nutch https://nutch.apache.org"
 
 WORKDIR /root/
 
-# Install dependencies
-RUN apk update
-RUN apk --no-cache add apache-ant bash git openjdk17
+# Install dependencies and shell init for JAVA_HOME
+RUN apk update && \
+    apk --no-cache add apache-ant bash git openjdk17 && \
+    echo 'export JAVA_HOME=/usr/lib/jvm/java-17-openjdk' >> "$HOME/.bashrc" && \
+    echo 'export JAVA_HOME=/usr/lib/jvm/java-17-openjdk' >> "$HOME/.ashrc"
 
-# Establish environment variables
-RUN echo 'export JAVA_HOME=/usr/lib/jvm/java-17-openjdk' >> $HOME/.bashrc
-RUN echo 'export JAVA_HOME=/usr/lib/jvm/java-17-openjdk' >> $HOME/.ashrc
 ENV JAVA_HOME='/usr/lib/jvm/java-17-openjdk'
 ENV NUTCH_HOME='/root/nutch_source/runtime/local'
 
 # Checkout and build the Nutch master branch (1.x)
 RUN git clone https://github.com/apache/nutch.git nutch_source && \
-     cd nutch_source && \
-     ant runtime && \
-     rm -rf build/ && \
-     rm -rf /root/.ivy2/
+    cd nutch_source && \
+    ant runtime && \
+    rm -rf build/ && \
+    rm -rf /root/.ivy2/
 
 # Create symlinks for runtime/local/bin/nutch and runtime/local/bin/crawl
-RUN ln -sf $NUTCH_HOME/bin/nutch /usr/local/bin/
-RUN ln -sf $NUTCH_HOME/bin/crawl /usr/local/bin/
+RUN ln -sf "$NUTCH_HOME/bin/nutch" /usr/local/bin/ && \
+    ln -sf "$NUTCH_HOME/bin/crawl" /usr/local/bin/
 
 RUN echo "Successfully built image, see https://s.apache.org/m5933 for guidance on running a container instance."
 

src/bin/crawl

@@ -23,7 +23,7 @@
 #
 # Options:
 #   -i|--index                            Indexes crawl results into a configured indexer
-#   -D <propery>=<value>                  A Nutch or Hadoop property to pass to Nutch calls overwriting
+#   -D <property>=<value>                 A Nutch or Hadoop property to pass to Nutch calls overwriting
 #                                         properties defined in configuration files, e.g.
 #                                           increase content limit to 2MB:
 #                                             -D http.content.limit=2097152
@@ -51,6 +51,8 @@
 #   --dedup-group <none|host|domain>       Deduplication group method [default: none]
 #
 
+## @audience private
+## @stability stable
 function __to_seconds() {
   NUMBER=$(echo $1 | tr -dc '0-9')
   MODIFIER=$(echo $1 | tr -dc '[^s|h|m|d]]')
@@ -73,6 +75,8 @@ function __to_seconds() {
   echo $SECONDS
 }
 
+## @audience private
+## @stability stable
 function __print_usage {
   echo "Usage: crawl [options] <crawl_dir> <num_rounds>"
   echo -e ""
@@ -129,7 +133,7 @@ NUM_THREADS=50
 SITEMAPS_FROM_HOSTDB_FREQUENCY=never
 DEDUP_GROUP=none
 
-while [[ $# > 0 ]]
+while [[ $# -gt 0 ]]
 do
     case $1 in
         -i|--index)
@@ -229,9 +233,12 @@ bin="`cd "$bin"; pwd`"
 
 # determines whether mode based on presence of job file
 mode=local
-if [ -f "${bin}"/../*nutch*.job ]; then
-  mode=distributed
-fi
+for f in "${bin}"/../*nutch*.job; do
+  if [ -f "$f" ]; then
+    mode=distributed
+    break
+  fi
+done
 if [[ "$mode" = "local" ]]; then
   if [[ "$NUM_FETCHERS" -ne 1 ]]; then
     echo "Ignoring configured number of fetchers (--num_fetchers): a single fetcher task is used when running in local mode."
@@ -247,28 +254,32 @@ commonOptions=("${HADOOP_PROPERTIES[@]}" -Dmapreduce.job.reduces=$NUM_TASKS -Dma
 if [ $mode = "distributed" ]; then
  if [ $(which hadoop | wc -l ) -eq 0 ]; then
     echo "Can't find Hadoop executable. Add HADOOP_COMMON_HOME/bin to the path or run in local mode."
-    exit -1;
+    exit 1;
  fi
 fi
 
 
+## @audience private
+## @stability stable
 function __bin_nutch {
     # run $bin/nutch, exit if exit value indicates error
 
-    echo "$bin/nutch $@" ;# echo command and arguments
+    echo "$bin/nutch" "$@" ;# echo command and arguments
     "$bin/nutch" "$@"
 
     RETCODE=$?
     if [ $RETCODE -ne 0 ]
     then
         echo "Error running:"
-        echo "  $bin/nutch $@"
+        echo "  $bin/nutch" "$@"
         echo "Failed with exit value $RETCODE."
         exit $RETCODE
     fi
 }
 
 # check if directory exists locally or on hdfs
+## @audience private
+## @stability stable
 function __directory_exists {
   if [[ "$mode" == local  &&  -d "$1" ]]; then
     return 0
@@ -279,6 +290,8 @@ function __directory_exists {
   fi
 }
 
+## @audience private
+## @stability stable
 function __update_hostdb {
   if __directory_exists "$CRAWL_PATH"/crawldb; then
     echo "Updating HostDB"
@@ -334,7 +347,7 @@ do
    generate_args=("${commonOptions[@]}" "$CRAWL_PATH"/crawldb "$CRAWL_PATH"/segments -topN $SIZE_FETCHLIST -numFetchers $NUM_FETCHERS -noFilter)
   fi
 
-  echo "$bin/nutch generate ${generate_args[@]}"
+  echo "$bin/nutch generate" "${generate_args[@]}"
   $bin/nutch generate "${generate_args[@]}"
   RETCODE=$?
   if [ $RETCODE -eq 0 ]; then
@@ -352,7 +365,7 @@ do
     fi
   else
     echo "Error running:"
-    echo "  $bin/nutch generate ${generate_args[@]}"
+    echo "  $bin/nutch generate" "${generate_args[@]}"
     echo "Failed with exit value $RETCODE."
     exit $RETCODE
   fi

src/bin/nutch

@@ -147,17 +147,18 @@ fi
 
 local=true
 
-# NUTCH_JOB 
-if [ -f "${NUTCH_HOME}"/*nutch*.job ]; then
-  local=false
-  for f in "$NUTCH_HOME"/*nutch*.job; do
+# NUTCH_JOB
+for f in "${NUTCH_HOME}"/*nutch*.job; do
+  if [ -f "$f" ]; then
+    local=false
     NUTCH_JOB="$f"
-  done
-  # cygwin path translation
-  if $cygwin; then
-	NUTCH_JOB="`cygpath -p -w "$NUTCH_JOB"`"
+    # cygwin path translation
+    if $cygwin; then
+      NUTCH_JOB="$(cygpath -p -w "$NUTCH_JOB")"
+    fi
+    break
   fi
-fi
+done
 
 JAVA="$JAVA_HOME/bin/java"
 JAVA_HEAP_MAX=-Xmx4096m
@@ -238,7 +239,7 @@ fi
 # figure out which class to run
 if [ "$COMMAND" = "crawl" ] ; then
   echo "Command $COMMAND is deprecated, please use bin/crawl instead"
-  exit -1
+  exit 1
 elif [ "$COMMAND" = "inject" ] ; then
   CLASS=org.apache.nutch.crawl.Injector
 elif [ "$COMMAND" = "generate" ] ; then
@@ -272,7 +273,7 @@ elif [ "$COMMAND" = "commoncrawldump" ] ; then
 elif [ "$COMMAND" = "solrindex" ] || [ "$COMMAND" = "solrdedup" ] || [ "$COMMAND" = "solrclean" ]; then
   REPLACEMENT="${COMMAND#solr}"
   echo "The command $COMMAND was replaced by the command $REPLACEMENT"
-  exit -1
+  exit 1
 elif [ "$COMMAND" = "index" ] ; then
   CLASS=org.apache.nutch.indexer.IndexingJob
 elif [ "$COMMAND" = "dedup" ] ; then
@@ -336,7 +337,7 @@ else
  # check that hadoop can be found on the path
  if [ $(which hadoop | wc -l ) -eq 0 ]; then
     echo "Can't find Hadoop executable. Add HADOOP_COMMON_HOME/bin to the path or run in local mode."
-    exit -1;
+    exit 1;
  fi
 fi
 

src/java/org/apache/nutch/crawl/CrawlDatum.java

@@ -419,7 +419,7 @@ public void set(CrawlDatum that) {
    * @param that an existing {@link CrawlDatum}
    * @return 1 if any one field (score, status, fetchTime, retries,
    * fetchInterval or modifiedTime) of the new {@link CrawlDatum}
-   * minus the correspoinding field of the existing {@link CrawlDatum}
+   * minus the corresponding field of the existing {@link CrawlDatum}
    * is greater than 0, otherwise return -1.
    */
   @Override

src/java/org/apache/nutch/crawl/CrawlDbReader.java

@@ -115,7 +115,7 @@ private void openReaders(String crawlDb, Configuration config)
     synchronized (this) {
       if (readers != null) {
         if (this.lastModified == lastModified) {
-          // CrawlDB not modified, re-use readers
+          // CrawlDB not modified, reuse readers
           return;
         } else {
           // CrawlDB modified, close and re-open readers

src/java/org/apache/nutch/crawl/LinkDbReader.java

@@ -87,7 +87,7 @@ public void openReaders() throws IOException {
     synchronized (this) {
       if (readers != null) {
         if (this.lastModified == lastModified) {
-          // CrawlDB not modified, re-use readers
+          // CrawlDB not modified, reuse readers
           return;
         } else {
           // CrawlDB modified, close and re-open readers

src/java/org/apache/nutch/hostdb/ResolverThread.java

@@ -107,7 +107,7 @@ private void initCounters() {
    */
   @Override
   public void run() {
-    // Resolve the host and act appropriatly
+    // Resolve the host and act appropriately
     try {
       // Throws an exception if host is not found
       @SuppressWarnings("unused")

src/java/org/apache/nutch/parse/Outlink.java

@@ -59,7 +59,7 @@ public void readFields(DataInput in) throws IOException {
   /**
    * Skips over one Outlink in the input.
    * @param in the {@link DataInput} tuple stream holding the
-   * toUrl and archor pair.
+   * toUrl and anchor pair.
    * @throws IOException if there is an error processing the {@link DataInput}
    */
   public static void skip(DataInput in) throws IOException {

src/java/org/apache/nutch/parse/OutlinkExtractor.java

@@ -61,7 +61,7 @@ public class OutlinkExtractor {
    * cases (postscript is a known example).
    * 
    * @param plainText
-   *          the plain text from wich URLs should be extracted.
+   *          the plain text from which URLs should be extracted.
    * @param conf a populated {@link Configuration}
    * @return Array of <code>Outlink</code>s within found in plainText
    */
@@ -75,7 +75,7 @@ public static Outlink[] getOutlinks(final String plainText,
    * extracted <code>Outlink</code>s
    * 
    * @param plainText
-   *          the plain text from wich URLs should be extracted.
+   *          the plain text from which URLs should be extracted.
    * @param anchor
    *          the anchor of the url
    * @param conf a populated {@link Configuration}

src/java/org/apache/nutch/parse/ParsePluginList.java

@@ -24,7 +24,7 @@
 /**
  * This class represents a natural ordering for which parsing plugin should get
  * called for a particular mimeType. It provides methods to store the
- * parse-plugins.xml data, and methods to retreive the name of the appropriate
+ * parse-plugins.xml data, and methods to retrieve the name of the appropriate
  * parsing plugin for a contentType.
  * 
  * @author mattmann

src/java/org/apache/nutch/parse/ParseResult.java

@@ -183,7 +183,7 @@ public boolean isSuccess() {
   /**
    * A convenience method which returns true if at least one of the parses is
    * successful. Parse success is determined by {@link ParseStatus#isSuccess()}.
-   * @return true if atleast one result is a success, false otherwise
+   * @return true if at least one result is a success, false otherwise
    */
   public boolean isAnySuccess() {
     for (Iterator<Entry<Text, Parse>> i = iterator(); i.hasNext();) {

src/java/org/apache/nutch/plugin/Extension.java

@@ -67,7 +67,7 @@ private void setExtensionPoint(String point) {
 
   /**
    * Returns a attribute value, that is setuped in the manifest file and is
-   * definied by the extension point xml schema.
+   * defined by the extension point xml schema.
    * 
    * @param pKey
    *          a key
@@ -109,7 +109,7 @@ public void addAttribute(String pKey, String pValue) {
   }
 
   /**
-   * Sets the Class that implement the concret extension and is only used until
+   * Sets the Class that implement the concrete extension and is only used until
    * model creation at system start up.
    * 
    * @param extensionClazz

src/java/org/apache/nutch/plugin/ExtensionPoint.java

@@ -103,7 +103,7 @@ private void setSchema(String pSchema) {
   }
 
   /**
-   * Install a coresponding extension to this extension point.
+   * Install a corresponding extension to this extension point.
    * 
    * @param extension the new {@link org.apache.nutch.plugin.Extension}
    * to install

src/java/org/apache/nutch/plugin/Plugin.java

@@ -58,7 +58,7 @@ public Plugin(PluginDescriptor pDescriptor, Configuration conf) {
    * used.
    * 
    * @throws PluginRuntimeException
-   *           If the startup was without successs.
+   *           If the startup was without success.
    */
   public void startUp() throws PluginRuntimeException {
   }