Major refactoring of package structure

src/main/java/io/archivesunleashed/mapreduce/WacInputFormat.java → src/main/java/io/archivesunleashed/data/ArchiveRecordInputFormat.java

@@ -15,13 +15,9 @@
  * limitations under the License.
  */
 
-package io.archivesunleashed.mapreduce;
+package io.archivesunleashed.data;
 
-import io.archivesunleashed.io.ArchiveRecordWritable.ArchiveFormat;
-import io.archivesunleashed.io.ArchiveRecordWritable;
-import java.io.BufferedInputStream;
-import java.io.IOException;
-import java.util.Iterator;
+import io.archivesunleashed.data.ArchiveRecordWritable.ArchiveFormat;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -42,10 +38,14 @@
 import org.archive.io.warc.WARCReader;
 import org.archive.io.warc.WARCReaderFactory.CompressedWARCReader;
 
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.util.Iterator;
+
 /**
  * Extends FileInputFormat for Web Archive Commons InputFormat.
  */
-public class WacInputFormat extends FileInputFormat<LongWritable,
+public class ArchiveRecordInputFormat extends FileInputFormat<LongWritable,
   ArchiveRecordWritable> {
   @Override
   public final RecordReader<LongWritable,

src/main/java/io/archivesunleashed/io/ArchiveRecordWritable.java → src/main/java/io/archivesunleashed/data/ArchiveRecordWritable.java

@@ -15,18 +15,17 @@
  * limitations under the License.
  */
 
-package io.archivesunleashed.io;
+package io.archivesunleashed.data;
 
-import io.archivesunleashed.data.ArcRecordUtils;
-import io.archivesunleashed.data.WarcRecordUtils;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
 import org.apache.hadoop.io.Writable;
 import org.archive.io.ArchiveRecord;
 import org.archive.io.arc.ARCRecord;
 import org.archive.io.warc.WARCRecord;
 
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
 /**
  * Implements Hadoop Writable for Archive Records.
  */

src/main/scala/io/archivesunleashed/spark/archive/io/ArchiveRecord.scala → src/main/scala/io/archivesunleashed/ArchiveRecord.scala

@@ -14,20 +14,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.archive.io
 
+package io.archivesunleashed
+
+import data.{ArcRecordUtils, WarcRecordUtils}
+import data.ArchiveRecordWritable.ArchiveFormat
+import matchbox.{ExtractDate, ExtractDomain, RemoveHttpHeader}
+import ExtractDate.DateComponent
 import java.text.SimpleDateFormat
 
+import io.archivesunleashed.data.ArchiveRecordWritable
 import org.apache.spark.SerializableWritable
 import org.archive.io.arc.ARCRecord
 import org.archive.io.warc.WARCRecord
-
 import org.archive.util.ArchiveUtils
-import io.archivesunleashed.data.{ArcRecordUtils, WarcRecordUtils}
-import io.archivesunleashed.io.ArchiveRecordWritable
-import io.archivesunleashed.io.ArchiveRecordWritable.ArchiveFormat
-import io.archivesunleashed.spark.matchbox.ExtractDate.DateComponent
-import io.archivesunleashed.spark.matchbox.{RemoveHttpHeader, ExtractDate, ExtractDomain}
 
 class ArchiveRecord(r: SerializableWritable[ArchiveRecordWritable]) extends Serializable {
   var arcRecord: ARCRecord = null

src/main/scala/io/archivesunleashed/spark/matchbox/ExtractEntities.scala → src/main/scala/io/archivesunleashed/app/ExtractEntities.scala

@@ -14,8 +14,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.app
 
+import io.archivesunleashed._
+import io.archivesunleashed.matchbox.{NER3Classifier, RemoveHTML}
 import org.apache.spark.SparkContext
 import org.apache.spark.rdd.RDD
 

src/main/scala/io/archivesunleashed/spark/matchbox/ExtractGraph.scala → src/main/scala/io/archivesunleashed/app/ExtractGraph.scala

@@ -14,14 +14,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.app
 
-import io.archivesunleashed.spark.archive.io.ArchiveRecord
+import io.archivesunleashed._
+import io.archivesunleashed.matchbox._
+import io.archivesunleashed.util.JsonUtils
 import org.apache.spark.graphx._
 import org.apache.spark.rdd.RDD
-import io.archivesunleashed.spark.matchbox.StringUtils._
-import io.archivesunleashed.spark.rdd.RecordRDD._
-import io.archivesunleashed.spark.utils.JsonUtil
 
 /**
   *
@@ -84,8 +83,8 @@ object ExtractGraph {
           "count" -> r._2)
       }
 
-      edgesCounted.map(r => JsonUtil.toJson(r)).saveAsTextFile(edgesPath)
-      graph.vertices.map(r => JsonUtil.toJson(r._2)).saveAsTextFile(verticesPath)
+      edgesCounted.map(r => JsonUtils.toJson(r)).saveAsTextFile(edgesPath)
+      graph.vertices.map(r => JsonUtils.toJson(r._2)).saveAsTextFile(verticesPath)
     }
   }
 }

src/main/scala/io/archivesunleashed/spark/matchbox/ExtractPopularImages.scala → src/main/scala/io/archivesunleashed/app/ExtractPopularImages.scala

@@ -14,13 +14,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.app
 
-import org.apache.spark.{RangePartitioner, SparkContext}
-import io.archivesunleashed.spark.rdd.RecordRDD._
+import io.archivesunleashed._
+import io.archivesunleashed.matchbox.{ComputeImageSize, ComputeMD5}
 import org.apache.spark.rdd.RDD
-import io.archivesunleashed.spark.archive.io.ArchiveRecord
-
+import org.apache.spark.{RangePartitioner, SparkContext}
 
 /**
   * Extract most popular images

src/main/scala/io/archivesunleashed/spark/matchbox/NERCombinedJson.scala → src/main/scala/io/archivesunleashed/app/NERCombinedJson.scala

@@ -14,16 +14,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.app
 
-import java.io.BufferedReader
-import java.io.BufferedWriter
-import java.io.InputStreamReader
-import java.io.OutputStreamWriter
+import java.io.{BufferedReader, BufferedWriter, InputStreamReader, OutputStreamWriter}
+
+import io.archivesunleashed.matchbox.NER3Classifier
+import io.archivesunleashed.util.JsonUtils
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs._
 import org.apache.spark.SparkContext
-import io.archivesunleashed.spark.utils.JsonUtil
+
 import scala.collection.mutable.MutableList
 import scala.util.Random
 
@@ -93,7 +93,7 @@ class NERCombinedJson extends Serializable {
           })
           .map(r => {
             val classifiedJson = NER3Classifier.classify(r._3)
-            val classifiedMap = JsonUtil.fromJson(classifiedJson)
+            val classifiedMap = JsonUtils.fromJson(classifiedJson)
             val classifiedMapCountTuples: Map[String, List[(String, Int)]] = classifiedMap.map {
               case (nerType, entities: List[String @unchecked]) => (nerType, entities.groupBy(identity).mapValues(_.size).toList)
             }
@@ -112,7 +112,7 @@ class NERCombinedJson extends Serializable {
             })
             nerRec.ner += ec
           })
-          JsonUtil.toJson(nerRec)
+          JsonUtils.toJson(nerRec)
         })
       })
       .saveAsTextFile(outputFile)

src/main/scala/io/archivesunleashed/spark/matchbox/WriteGEXF.scala → src/main/scala/io/archivesunleashed/app/WriteGEXF.scala

@@ -14,15 +14,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.app
+
+import io.archivesunleashed.matchbox._
 
-import java.io.BufferedWriter
-import java.io.OutputStreamWriter
 import java.nio.charset.StandardCharsets
-import java.nio.file.Files
-import java.nio.file.Paths
+import java.nio.file.{Files, Paths}
+
 import org.apache.spark.rdd.RDD
-import StringUtils._
 
 
 /**

src/main/scala/io/archivesunleashed/spark/matchbox/WriteGraphML.scala → src/main/scala/io/archivesunleashed/app/WriteGraphML.scala

@@ -14,15 +14,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.app
+
+import io.archivesunleashed.matchbox._
 
-import java.io.BufferedWriter
-import java.io.OutputStreamWriter
 import java.nio.charset.StandardCharsets
-import java.nio.file.Files
-import java.nio.file.Paths
+import java.nio.file.{Files, Paths}
+
 import org.apache.spark.rdd.RDD
-import StringUtils._
 
 
 

src/main/scala/io/archivesunleashed/spark/matchbox/ComputeImageSize.scala → src/main/scala/io/archivesunleashed/matchbox/ComputeImageSize.scala

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.matchbox
 
 import java.io.ByteArrayInputStream
 import javax.imageio.ImageIO

src/main/scala/io/archivesunleashed/spark/matchbox/ComputeMD5.scala → src/main/scala/io/archivesunleashed/matchbox/ComputeMD5.scala

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.matchbox
 
 import java.security.MessageDigest
 

src/main/scala/io/archivesunleashed/spark/matchbox/DetectLanguage.scala → src/main/scala/io/archivesunleashed/matchbox/DetectLanguage.scala

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.matchbox
 
 import org.apache.tika.language.LanguageIdentifier
 

src/main/scala/io/archivesunleashed/spark/matchbox/DetectMimeTypeTika.scala → src/main/scala/io/archivesunleashed/matchbox/DetectMimeTypeTika.scala

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.matchbox
 
 import java.io.ByteArrayInputStream
 

src/main/scala/io/archivesunleashed/spark/matchbox/ExtractAtMentions.scala → src/main/scala/io/archivesunleashed/matchbox/ExtractAtMentions.scala

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.matchbox
 
 object ExtractAtMentions {
   val pattern = """@[A-Za-z_0-9]+""".r

src/main/scala/io/archivesunleashed/spark/matchbox/ExtractBoilerpipeText.scala → src/main/scala/io/archivesunleashed/matchbox/ExtractBoilerpipeText.scala

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.matchbox
 
 import java.io.IOException
 import de.l3s.boilerpipe.extractors.DefaultExtractor

src/main/scala/io/archivesunleashed/spark/matchbox/ExtractDate.scala → src/main/scala/io/archivesunleashed/matchbox/ExtractDate.scala

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.matchbox
 
 /**
   * Simple wrapper for getting different parts of a date

src/main/scala/io/archivesunleashed/spark/matchbox/ExtractDomain.scala → src/main/scala/io/archivesunleashed/matchbox/ExtractDomain.scala

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.matchbox
 
 import java.net.URL
 

src/main/scala/io/archivesunleashed/spark/matchbox/ExtractHashtags.scala → src/main/scala/io/archivesunleashed/matchbox/ExtractHashtags.scala

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.matchbox
 
 object ExtractHashtags {
   val pattern = """#[^ ]+""".r

src/main/scala/io/archivesunleashed/spark/matchbox/ExtractImageLinks.scala → src/main/scala/io/archivesunleashed/matchbox/ExtractImageLinks.scala

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.matchbox
 
 import java.io.IOException
 

src/main/scala/io/archivesunleashed/spark/matchbox/ExtractLinks.scala → src/main/scala/io/archivesunleashed/matchbox/ExtractLinks.scala

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.matchbox
 
 import java.io.IOException
 

src/main/scala/io/archivesunleashed/spark/matchbox/ExtractTextFromPDFs.scala → src/main/scala/io/archivesunleashed/matchbox/ExtractTextFromPDFs.scala

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.matchbox
 
 import java.io.ByteArrayInputStream
 

src/main/scala/io/archivesunleashed/spark/matchbox/ExtractUrls.scala → src/main/scala/io/archivesunleashed/matchbox/ExtractUrls.scala

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.matchbox
 
 object ExtractUrls {
   val pattern = """https?://[^ ]+""".r

src/main/scala/io/archivesunleashed/spark/matchbox/NER3Classifier.scala → src/main/scala/io/archivesunleashed/matchbox/NER3Classifier.scala

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.archivesunleashed.spark.matchbox
+package io.archivesunleashed.matchbox
 
 import java.util