Remove .keepValidPages() on .all() Python implementation. (#503)

- Resolves #502
- Update test
- Run GitHub actions on correct branch

.github/workflows/java-formatter.yml

@@ -2,7 +2,7 @@ name: Java Formatter (Google Java Format)
 
 on:
   pull_request:
-    branches: [master]
+    branches: [main]
 
 jobs:
   formatting:

.github/workflows/python-formatter.yml

@@ -2,7 +2,7 @@ name: Python Formatter (isort & black)
 
 on:
   pull_request:
-    branches: [master]
+    branches: [main]
 
 jobs:
   formatting:

.github/workflows/scala-formatter.yml

@@ -2,7 +2,7 @@ name: Run Scala Formatter (scalafmt)
 
 on:
   pull_request:
-    branches: [master]
+    branches: [main]
 
 jobs:
   scalafmt-lint:

src/main/scala/io/archivesunleashed/df/DataFrameLoader.scala

@@ -27,7 +27,6 @@ class DataFrameLoader(sc: SparkContext) {
   def all(path: String): DataFrame = {
     RecordLoader
       .loadArchives(path, sc)
-      .keepValidPages()
       .all()
   }
 

src/test/scala/io/archivesunleashed/df/DataFrameLoaderTest.scala

@@ -123,8 +123,10 @@ class DataFrameLoaderTest extends FunSuite with BeforeAndAfter {
     assert(r_10.getAs[String](md5) == "e483512b65ba44d71e843c57de2adeb7")
 
     val r_11 = all.select(url, mime_type).take(1)(0)
-    assert(r_11.getAs[String](url) == "http://www.archive.org/")
-    assert(r_11.getAs[String](mime_type) == "text/html")
+    assert(
+      r_11.getAs[String](url) == "filedesc://IAH-20080430204825-00000-blackbook.arc"
+    )
+    assert(r_11.getAs[String](mime_type) == "text/plain")
   }
 
   after {