greebie
commented
6 years ago import io.archivesunleashed.spark.matchbox._
import io.archivesunleashed.spark.rdd.RecordRDD._
import StringUtils._
import java.time.Instant
def timed(f: => Unit) = {
val start = System.currentTimeMillis()
f
val end = System.currentTimeMillis()
println("Elapsed Time: " + (end - start))
}
timed {
println("Get urls and count, taking 3.")
val r = RecordLoader.loadArchives("/shared/au/example.arc.gz", sc)
.map (r => ExtractDomain(r.getUrl))
.countItems()
println(r.take(3).deep.mkString("\n"))
}
timed {
println("Get Hyperlinks from text and site and count, filtering out counts < 5, taking 3.")
val links = RecordLoader.loadArchives("/shared/au/example.arc.gz", sc)
.flatMap(r => ExtractLinks(r.getUrl, r.getContentString))
.map(r => (ExtractDomain(r._1).removePrefixWWW(), ExtractDomain(r._2).removePrefixWWW()))
.filter(r => r._1 != "" && r._2 != "")
.countItems()
.filter(r => r._2 > 5)
println(links.take(3).deep.mkString("\n"))
}
timed {
println("Get links from text and site, group by date and count, filtering out counts < 5, taking 3.")
val crawlDateGroup = RecordLoader.loadArchives("/shared/au/example.arc.gz", sc)
.map(r => (r.getCrawlDate, ExtractLinks(r.getUrl, r.getContentString)))
.flatMap(r => r._2.map(f => (r._1, ExtractDomain(f._1).replaceAll("^\\s*www\\.", ""), ExtractDomain(f._2).replaceAll("^\\s*www\\.", ""))))
.filter(r => r._2 != "" && r._3 != "")
.countItems()
.filter(r => r._2 > 5)
println(crawlDateGroup.take(3).deep.mkString("\n"))
}
timed {
println ("Extract text, taking 3 examples.")
val text = RecordLoader.loadArchives("/shared/au/example.arc.gz", sc)
.map(r => (r.getCrawlDate, r.getDomain, r.getUrl, RemoveHTML(r.getContentString)))
println(text.take(3).deep.mkString("\n"))
}
timed {
println ("Extract image urls, taking 3.")
val images = RecordLoader.loadArchives("/shared/au/example.arc.gz", sc)
.flatMap(r => ExtractImageLinks(r.getUrl, r.getContentString))
.countItems()
println (images.take(3).deep.mkString("\n"))
}
ruebot
ianmilligan1
eromoe
We are comparing the timings of Scala vs Python to help us make informed decisions on the migration.