Apache Nutch 2.2, MySQL & Solr 5.2.1 Tutorial¶

# Create our working dir
mkdir ~/Desktop/Nutch
cd ~/Desktop/Nutch

# Grab all our files (correct versions!)
curl -O http://archive.apache.org/dist/nutch/2.2/apache-nutch-2.2-src.tar.gz
curl -O http://archive.apache.org/dist/lucene/solr/5.2.1/solr-5.2.1.tgz

# Unzip and rename them (for convenience!)
tar -xzf apache-nutch-2.2-src.tar.gz && mv apache-nutch-2.2 nutch
tar -xzf solr-5.2.1.tgz && mv solr-5.2.1 solr

# Delete command if you wanna start again!
# rm -rf nutch hbase solr

# Move our source files to another directory to keep things clean!
mkdir gzips && mv apache-nutch-2.2-src.tar.gz solr-5.2.1.tgz gzips

# Your working directory should now look like this:
# .
# ├── gzips
# ├── nutch
# └── solr

export JAVA_HOME="$(/usr/libexec/java_home -v 1.8)"
export NUTCH_JAVA_HOME="$(/usr/libexec/java_home -v 1.8)"

~/Desktop/Nutch/solr/bin/solr start

# Create a new core
~/Desktop/Nutch/solr/bin/solr create_core -c demo

open ~/Desktop/Nutch/nutch/conf/nutch-site.xml

# Place the following content's into it

# Ensure hbase and solr are started!
~/Desktop/Nutch/solr/bin/solr start

# We'll edit the regex filter to only crawl the apache nutch domain
open ~/Desktop/Nutch/nutch/runtime/local/conf/regex-urlfilter.txt

# Add the following line before the `accept anything else +.`
+^http://nutch.apache.org

# Insert the url into our `seed` list
mkdir -p ~/Desktop/Nutch/nutch/runtime/local/urls/
echo "http://nutch.apache.org" > ~/Desktop/Nutch/nutch/runtime/local/urls/seed.txt

# Change our working dir
cd ~/Desktop/Nutch/nutch/runtime/local/

# Inject the url's from our seed list (in the directory named `urls`)
# This injects into our Hbase database (as that's what we have specified)
bin/nutch inject urls

# Generate a segment which tells Nutch which and how to fetch the urls.
# Creates folder <#timestamp#> under the <segments_dir>
# Inside that dir creates the folder "crawl_generate" where Nutch puts list of URLs to be fetched.
# -topN = How many pages to crawl this depth
# This will only contain 1 URL the first round we execute
bin/nutch generate -topN 10

# Fetch the URLs we generated in the previous step.
# Nutch places the data into the "content" and "crawl_fetch" folders inside the <segments_dir>/#timestamp# directory.
bin/nutch fetch -all

# Instruct Nutch to parse the fetched data which is placed into three folders
# "crawl_parse", "parse_data" and "parse_text" inside the <segments_dir>/#timestamp# directory.
# We can write a custom ParserFilter extension to parse content how we like
bin/nutch parse -all

# Update the Database with the new urls
bin/nutch updatedb

# Finally let's push everything to solr (and to our demo core)
bin/nutch solrindex http://localhost:8983/solr/demo -all

# Running everything again
cd ~/Desktop/Nutch/nutch/runtime/local/
bin/nutch inject urls
bin/nutch generate -topN 10
bin/nutch fetch -all
bin/nutch parse -all
bin/nutch updatedb
bin/nutch solrindex http://localhost:8983/solr/demo -all

# One liner
bin/nutch inject urls && bin/nutch generate -topN 10 && bin/nutch fetch -all && bin/nutch parse -all && bin/nutch updatedb
bin/nutch solrindex http://localhost:8983/solr/demo -all

# Clearing the solr database
rm -rf ~/Desktop/Nutch/solr/server/solr/demo/data && ~/Desktop/Nutch/solr/bin/solr restart
cd ~/Desktop/Nutch/solr && bin/solr restart
bin/solr create_core -c demo

# Clearing the hbase database
# The hbase db store the url's for nutch, you may want to clear this also
~/Desktop/Nutch/hbase/bin/hbase shell
disable 'webpage'
drop 'webpage'