From 362f194e4173ff82459adf214b82325b5055f7bc Mon Sep 17 00:00:00 2001 From: "v.bazarevsky" Date: Mon, 18 Apr 2016 11:27:45 +0300 Subject: [PATCH 1/4] Initial commit --- .gitignore | 17 +++++++++++++++++ README.md | 1 + 2 files changed, 18 insertions(+) create mode 100644 .gitignore create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c58d83b --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +*.class +*.log + +# sbt specific +.cache +.history +.lib/ +dist/* +target/ +lib_managed/ +src_managed/ +project/boot/ +project/plugins/project/ + +# Scala-IDE specific +.scala_dependencies +.worksheet diff --git a/README.md b/README.md new file mode 100644 index 0000000..32a00f8 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# scalalab3_lyricsengine \ No newline at end of file From 1a339fc71c94bf49ad87c7af727da836fb5ed167 Mon Sep 17 00:00:00 2001 From: Aliaksandr Bialkevich Date: Thu, 21 Apr 2016 16:54:32 +0300 Subject: [PATCH 2/4] Create project structure --- .gitignore | 3 +++ build.sbt | 10 ++++++++++ data_analyzer/build.sbt | 5 +++++ http_api/build.sbt | 5 +++++ raw_data_processor/build.sbt | 5 +++++ raw_data_processor/project/build.properties | 1 + raw_data_processor/project/plugins.sbt | 1 + .../scala/scalalab3/lyricsengine/reader/Reader.scala | 4 ++++ 8 files changed, 34 insertions(+) create mode 100644 build.sbt create mode 100644 data_analyzer/build.sbt create mode 100644 http_api/build.sbt create mode 100644 raw_data_processor/build.sbt create mode 100644 raw_data_processor/project/build.properties create mode 100644 raw_data_processor/project/plugins.sbt create mode 100644 raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/Reader.scala diff --git a/.gitignore b/.gitignore index c58d83b..0f79af5 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ project/plugins/project/ # Scala-IDE specific .scala_dependencies .worksheet + +# IntelliJ IDEA +/.idea \ No newline at end of file diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..d3b1459 --- /dev/null +++ b/build.sbt @@ -0,0 +1,10 @@ +name := "lyrics-engine" + +version := "1.0" + +scalaVersion := "2.11.8" + + +lazy val raw_data_processor = project in file("raw_data_processor") +lazy val data_analyzer = project in file("data_analyzer") +lazy val http_api = project in file("http_api") diff --git a/data_analyzer/build.sbt b/data_analyzer/build.sbt new file mode 100644 index 0000000..3b20f89 --- /dev/null +++ b/data_analyzer/build.sbt @@ -0,0 +1,5 @@ +name := "data_analyzer" + +version := "1.0" + +scalaVersion := "2.11.8" \ No newline at end of file diff --git a/http_api/build.sbt b/http_api/build.sbt new file mode 100644 index 0000000..761a2ad --- /dev/null +++ b/http_api/build.sbt @@ -0,0 +1,5 @@ +name := "http_api" + +version := "1.0" + +scalaVersion := "2.11.8" \ No newline at end of file diff --git a/raw_data_processor/build.sbt b/raw_data_processor/build.sbt new file mode 100644 index 0000000..3eeb344 --- /dev/null +++ b/raw_data_processor/build.sbt @@ -0,0 +1,5 @@ +name := "raw_data_processor" + +version := "1.0" + +scalaVersion := "2.11.8" \ No newline at end of file diff --git a/raw_data_processor/project/build.properties b/raw_data_processor/project/build.properties new file mode 100644 index 0000000..d638b4f --- /dev/null +++ b/raw_data_processor/project/build.properties @@ -0,0 +1 @@ +sbt.version = 0.13.8 \ No newline at end of file diff --git a/raw_data_processor/project/plugins.sbt b/raw_data_processor/project/plugins.sbt new file mode 100644 index 0000000..14a6ca1 --- /dev/null +++ b/raw_data_processor/project/plugins.sbt @@ -0,0 +1 @@ +logLevel := Level.Warn \ No newline at end of file diff --git a/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/Reader.scala b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/Reader.scala new file mode 100644 index 0000000..f74205d --- /dev/null +++ b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/Reader.scala @@ -0,0 +1,4 @@ +package scalalab3.lyricsengine.reader + +object Reader { +} From fa9bd4ca2c7dd9aae7d04039705e6b64cf54626a Mon Sep 17 00:00:00 2001 From: Aliaksandr Bialkevich Date: Thu, 21 Apr 2016 21:43:22 +0300 Subject: [PATCH 3/4] MSD data file reader. --- build.sbt | 2 + raw_data_processor/build.sbt | 6 ++- .../src/main/resources/application.conf | 1 + .../processor/SongProcessor.scala | 7 +++ .../processor/WordsProcessor.scala | 7 +++ .../scalalab3/lyricsengine/reader/MSD.scala | 9 ++++ .../lyricsengine/reader/Reader.scala | 45 +++++++++++++++++++ .../src/test/resources/test_msd.txt | 20 +++++++++ .../src/test/scala/ReaderTest.scala | 9 ++++ 9 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 raw_data_processor/src/main/resources/application.conf create mode 100644 raw_data_processor/src/main/scala/scalalab3/lyricsengine/processor/SongProcessor.scala create mode 100644 raw_data_processor/src/main/scala/scalalab3/lyricsengine/processor/WordsProcessor.scala create mode 100644 raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/MSD.scala create mode 100644 raw_data_processor/src/test/resources/test_msd.txt create mode 100644 raw_data_processor/src/test/scala/ReaderTest.scala diff --git a/build.sbt b/build.sbt index d3b1459..76be9ef 100644 --- a/build.sbt +++ b/build.sbt @@ -8,3 +8,5 @@ scalaVersion := "2.11.8" lazy val raw_data_processor = project in file("raw_data_processor") lazy val data_analyzer = project in file("data_analyzer") lazy val http_api = project in file("http_api") + +// TODO Setup visibility common dependencies. \ No newline at end of file diff --git a/raw_data_processor/build.sbt b/raw_data_processor/build.sbt index 3eeb344..3c19e19 100644 --- a/raw_data_processor/build.sbt +++ b/raw_data_processor/build.sbt @@ -2,4 +2,8 @@ name := "raw_data_processor" version := "1.0" -scalaVersion := "2.11.8" \ No newline at end of file +scalaVersion := "2.11.8" + + +libraryDependencies += "com.typesafe" % "config" % "1.3.0" +libraryDependencies += "org.scalatest" % "scalatest_2.11" % "2.2.6" diff --git a/raw_data_processor/src/main/resources/application.conf b/raw_data_processor/src/main/resources/application.conf new file mode 100644 index 0000000..dfa71cc --- /dev/null +++ b/raw_data_processor/src/main/resources/application.conf @@ -0,0 +1 @@ +path_to_msd = "C:\\Temp\\mxm_dataset_train.txt" \ No newline at end of file diff --git a/raw_data_processor/src/main/scala/scalalab3/lyricsengine/processor/SongProcessor.scala b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/processor/SongProcessor.scala new file mode 100644 index 0000000..cb083ab --- /dev/null +++ b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/processor/SongProcessor.scala @@ -0,0 +1,7 @@ +package scalalab3.lyricsengine.processor + +/** + * Performs operations above songs. + */ +object SongProcessor { +} diff --git a/raw_data_processor/src/main/scala/scalalab3/lyricsengine/processor/WordsProcessor.scala b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/processor/WordsProcessor.scala new file mode 100644 index 0000000..8e6e3c5 --- /dev/null +++ b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/processor/WordsProcessor.scala @@ -0,0 +1,7 @@ +package scalalab3.lyricsengine.processor + +/** + * Performs operations above words. + */ +object WordsProcessor { +} diff --git a/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/MSD.scala b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/MSD.scala new file mode 100644 index 0000000..c86a53e --- /dev/null +++ b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/MSD.scala @@ -0,0 +1,9 @@ +package scalalab3.lyricsengine.reader + + +/** + * That class represent data from Million Song Dataset (MSD). + */ +case class MSD(words: Seq[String], songs: Seq[Song]) + +case class Song(trackId: String, mxmTrackId: String, words: Map[Int, Int]) \ No newline at end of file diff --git a/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/Reader.scala b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/Reader.scala index f74205d..0a7f545 100644 --- a/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/Reader.scala +++ b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/Reader.scala @@ -1,4 +1,49 @@ package scalalab3.lyricsengine.reader +import com.typesafe.config.ConfigFactory + +import scala.io.Source + object Reader { + val conf = ConfigFactory.load(); + + def main(args: Array[String]) { + print(extract()) + } + + def extract(): MSD = { + val path = conf.getString("path_to_msd") + + val pair = read(path) + + val words = extractWords(pair._1) + val songs = extractSongs(pair._2) + + new MSD(words, songs) + } + + private def read(path: String) = { + val lines = Source.fromFile(path).getLines().toSeq + val words = lines(17) + val songs = lines.slice(18, lines.size) + (words, songs) + } + + private def extractWords(words: String) = words.substring(1).split(","); + + private def extractSongs(songsData: Seq[String]) = songsData.map(songData => { + val elements = songData.split(",") + val trackId = elements(0) + val mxmTrackId = elements(1) + val words_map = extractSongWords(elements.slice(2, elements.size)) + new Song(trackId, mxmTrackId, words_map); + }) + + private def extractSongWords(words: Array[String]): Map[Int, Int] = words.toSeq.map(source_pair => { + val pair = source_pair.split(":") + assert(pair.size == 2) + pair(0).toInt -> pair(1).toInt + }).toMap; } + + diff --git a/raw_data_processor/src/test/resources/test_msd.txt b/raw_data_processor/src/test/resources/test_msd.txt new file mode 100644 index 0000000..640ea37 --- /dev/null +++ b/raw_data_processor/src/test/resources/test_msd.txt @@ -0,0 +1,20 @@ +# TRAINING SET +# MusiXmatch dataset, the official lyrics dataset +# of the Million Song Dataset +# file created on Tue Mar 29 04:28:44 2011 +# contact: T. Bertin-Mahieux (Columbia University) +# tb2332@columbia.edu +# also: http://labrosa.ee.columbia.edu/millionsong/musixmatch +# http://www.musixmatch.com +# FORMAT: +# # - comment, to ignore +# % - list of top words, comma-separated +# - normal line, contains track_id, mxm track id, +# then word count for each of the top words, comma-separated +# word count is in sparse format -> ...,:,... +# starts at 1 (not zero!) +# All our work is done using UTF-8 encoding. +# enjoy! +%i,the,you,to,and,a,me,it,not,in,my,is,of,your,that,do,on,are,we,am,will,all,for,no,be,have,love,so,know,this,but,with,what,just,when,like,now,que,time,can,come,de,there,go,up,oh,la,one,they,out,down,get,she,was,see,if,got,never,from,he,feel,want,let,make,way,say,take,would,as,ca,day,at,babi,away +TRZZZYV128F92E996D,6849828,1:10,2:6,3:20,5:2,7:30,8:1,9:6,10:3,11:19,12:1,13:16,14:6,15:1,18:4,20:2,21:2,23:1,24:4,26:4,31:3,35:1,37:2,50:15,52:21,56:1,57:3,59:3,63:6,68:2,74:3,75:3,84:1,92:1,94:3,100:1,101:2,111:2,121:3,123:5,142:3,144:1,155:1,162:31,167:6,184:3,203:3,216:1,227:1,240:2,261:1,358:1,368:2,372:4,382:3,403:2,415:2,434:1,467:3,485:1,571:1,592:3,627:4,672:6,1023:1,1123:1,1147:1,1247:1,1262:1,1294:1,1375:2,1391:15,1542:1,1619:1,2386:1,4817:3 +TRZZZYX128F92D32C6,681124,1:4,2:18,4:3,5:6,6:9,7:1,8:5,9:1,10:12,11:2,12:9,13:5,15:4,17:2,18:4,19:4,21:2,22:3,25:3,29:2,30:3,32:5,34:3,35:2,37:2,39:3,43:2,44:7,45:2,49:2,50:2,52:8,54:2,56:1,59:1,63:10,64:1,66:1,71:2,72:1,78:1,79:1,84:2,91:1,92:2,95:1,96:1,103:1,110:1,115:1,127:1,134:4,135:2,136:3,137:1,138:2,140:1,146:2,150:1,152:1,192:1,206:1,207:1,222:3,239:1,248:2,258:1,270:1,274:1,283:1,294:2,300:1,305:1,318:1,337:1,338:1,346:1,347:1,349:1,371:1,398:1,406:1,445:1,451:1,459:1,478:1,487:1,492:1,502:3,516:1,539:1,548:2,549:1,553:1,592:2,617:1,633:1,666:1,681:1,727:1,774:1,775:7,789:1,811:1,844:1,915:1,942:1,950:1,979:1,1008:1,1040:1,1080:9,1142:1,1232:1,1366:1,1409:1,1412:1,1537:1,1545:1,1597:1,1841:2,1861:1,1901:1,1907:1,2063:1,2097:1,2167:1,2198:1,2221:1,2468:1,2498:1,2595:1,2698:1,2975:1,2990:1,2996:1,3256:1,3267:1,3316:2,3355:1,4198:1,4356:1,4738:1,4845:1 \ No newline at end of file diff --git a/raw_data_processor/src/test/scala/ReaderTest.scala b/raw_data_processor/src/test/scala/ReaderTest.scala new file mode 100644 index 0000000..2d2d1d0 --- /dev/null +++ b/raw_data_processor/src/test/scala/ReaderTest.scala @@ -0,0 +1,9 @@ +import org.scalatest.{FlatSpec, Matchers} + +class ReaderTest extends FlatSpec with Matchers { + + "A MSD" should "contains" in { + // val raw = Source.fromInputStream(getClass.getResourceAsStream("/test_msd.txt")).mkString + // Reader.extract("/test_msd.txt") + } +} From d3b1a7d9cfb8198369bc8eb00a2cc5fede25a00d Mon Sep 17 00:00:00 2001 From: Aliaksandr Bialkevich Date: Sat, 23 Apr 2016 00:12:58 +0300 Subject: [PATCH 4/4] Resolve remark. Start writing Repositories layer. Other improvements. --- data_analyzer/build.sbt | 5 +- raw_data_processor/build.sbt | 17 +++++++ .../src/main/resources/application.conf | 13 ++++- .../scalalab3/lyricsengine/reader/MSD.scala | 9 ---- .../reader/MillionSongDataSubSetReader.scala | 49 +++++++++++++++++++ .../lyricsengine/reader/Reader.scala | 49 ------------------- .../MillionSongDataSubSetRepository.scala | 18 +++++++ .../lyricsengine/system/ProjectConfig.scala | 24 +++++++++ .../lyricsengine/system/ProjectLogger.scala | 8 +++ .../src/test/resources/test_msd.txt | 6 +-- .../MillionSongDataSubSetReaderTest.scala | 30 ++++++++++++ .../src/test/scala/ReaderTest.scala | 9 ---- 12 files changed, 165 insertions(+), 72 deletions(-) delete mode 100644 raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/MSD.scala create mode 100644 raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/MillionSongDataSubSetReader.scala delete mode 100644 raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/Reader.scala create mode 100644 raw_data_processor/src/main/scala/scalalab3/lyricsengine/repositories/MillionSongDataSubSetRepository.scala create mode 100644 raw_data_processor/src/main/scala/scalalab3/lyricsengine/system/ProjectConfig.scala create mode 100644 raw_data_processor/src/main/scala/scalalab3/lyricsengine/system/ProjectLogger.scala create mode 100644 raw_data_processor/src/test/scala/MillionSongDataSubSetReaderTest.scala delete mode 100644 raw_data_processor/src/test/scala/ReaderTest.scala diff --git a/data_analyzer/build.sbt b/data_analyzer/build.sbt index 3b20f89..306924f 100644 --- a/data_analyzer/build.sbt +++ b/data_analyzer/build.sbt @@ -2,4 +2,7 @@ name := "data_analyzer" version := "1.0" -scalaVersion := "2.11.8" \ No newline at end of file +scalaVersion := "2.11.8" + +//libraryDependencies += "org.apache.spark" % "spark-core_2.10" % "1.6.1" +//libraryDependencies += "org.apache.spark" % "spark-mllib_2.10" % "1.6.1" diff --git a/raw_data_processor/build.sbt b/raw_data_processor/build.sbt index 3c19e19..d080a34 100644 --- a/raw_data_processor/build.sbt +++ b/raw_data_processor/build.sbt @@ -4,6 +4,23 @@ version := "1.0" scalaVersion := "2.11.8" +// -------- -------- Dependencies: -------- -------- +// -------- Config: +// https://github.com/typesafehub/config libraryDependencies += "com.typesafe" % "config" % "1.3.0" + +// -------- Logger: +// http://logback.qos.ch/ +libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.1.7" +// https://github.com/typesafehub/scala-logging +libraryDependencies += "com.typesafe.scala-logging" %% "scala-logging" % "3.4.0" + +// -------- Test: +// http://www.scalatest.org/ libraryDependencies += "org.scalatest" % "scalatest_2.11" % "2.2.6" + +// -------- DB: +// mongo: +// http://mongodb.github.io/mongo-scala-driver/1.1/ +libraryDependencies += "org.mongodb.scala" % "mongo-scala-driver_2.11" % "1.1.0" \ No newline at end of file diff --git a/raw_data_processor/src/main/resources/application.conf b/raw_data_processor/src/main/resources/application.conf index dfa71cc..1b66bcf 100644 --- a/raw_data_processor/src/main/resources/application.conf +++ b/raw_data_processor/src/main/resources/application.conf @@ -1 +1,12 @@ -path_to_msd = "C:\\Temp\\mxm_dataset_train.txt" \ No newline at end of file +// ---- ---- Million Song Dataset data: +// subset from: http://labrosa.ee.columbia.edu/millionsong/sites/default/files/AdditionalFiles/mxm_dataset_train.txt.zip +msd.pathToMsd = "C:\\Temp\\mxm_dataset_train.txt" + +// ---- ---- DB: +// mongoDB: +// Nice mongo admin: https://github.com/mrvautin/adminMongo +// Short introduction to mongoDB installation: +// http://stackoverflow.com/questions/26585433/mongodb-failed-to-connect-to-127-0-0-127017-reason-errno10061 +mongoDb.host = localhost +mongoDb.port = 27017 +mongoDb.dbName = liric \ No newline at end of file diff --git a/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/MSD.scala b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/MSD.scala deleted file mode 100644 index c86a53e..0000000 --- a/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/MSD.scala +++ /dev/null @@ -1,9 +0,0 @@ -package scalalab3.lyricsengine.reader - - -/** - * That class represent data from Million Song Dataset (MSD). - */ -case class MSD(words: Seq[String], songs: Seq[Song]) - -case class Song(trackId: String, mxmTrackId: String, words: Map[Int, Int]) \ No newline at end of file diff --git a/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/MillionSongDataSubSetReader.scala b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/MillionSongDataSubSetReader.scala new file mode 100644 index 0000000..e0cdec1 --- /dev/null +++ b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/MillionSongDataSubSetReader.scala @@ -0,0 +1,49 @@ +package scalalab3.lyricsengine.reader + +import scala.io.Source + +object MillionSongDataSubSetReader { + def extract(path: String): MillionSongDataSubSet = { + val wordsSongsPair = readRawData(path) + new MillionSongDataSubSet(extractWords(wordsSongsPair._1), extractSongs(wordsSongsPair._2)) + } + + private def readRawData(path: String) = { + val lines = Source.fromFile(path).getLines().toSeq + // 17 - Line that contains the words. + val words = lines(17) + // 18 - First line that contains the data by songs. + val songs = lines.slice(18, lines.size) + words -> songs + } + + private def extractWords(words: String) = words.substring(1).split(","); + + private def extractSongs(songsData: Seq[String]) = songsData.map( + songData => { + val elements = songData.split(",") + val trackId = elements(0) + val mxmTrackId = elements(1) + val words_map = extractSongWords(elements.slice(2, elements.size)) + new Song(trackId, mxmTrackId, words_map); + }).toSeq + + private def extractSongWords(words: Array[String]) = words.toSeq.map( + pairInStringForm => { + val pair = pairInStringForm.split(":") + val wordIndex = pair(0).toInt + val count = pair(1).toInt + wordIndex -> count + }).toMap +} + +/** + * That class represent subset of data from Million Song Dataset (MSD). + * http://labrosa.ee.columbia.edu/millionsong/sites/default/files/AdditionalFiles/mxm_dataset_train.txt.zip + */ +case class MillionSongDataSubSet(words: Seq[String], songs: Seq[Song]) + +case class Song(trackId: String, mxmTrackId: String, words: Map[Int, Int]) + + + diff --git a/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/Reader.scala b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/Reader.scala deleted file mode 100644 index 0a7f545..0000000 --- a/raw_data_processor/src/main/scala/scalalab3/lyricsengine/reader/Reader.scala +++ /dev/null @@ -1,49 +0,0 @@ -package scalalab3.lyricsengine.reader - -import com.typesafe.config.ConfigFactory - -import scala.io.Source - -object Reader { - val conf = ConfigFactory.load(); - - def main(args: Array[String]) { - print(extract()) - } - - def extract(): MSD = { - val path = conf.getString("path_to_msd") - - val pair = read(path) - - val words = extractWords(pair._1) - val songs = extractSongs(pair._2) - - new MSD(words, songs) - } - - private def read(path: String) = { - val lines = Source.fromFile(path).getLines().toSeq - val words = lines(17) - val songs = lines.slice(18, lines.size) - (words, songs) - } - - private def extractWords(words: String) = words.substring(1).split(","); - - private def extractSongs(songsData: Seq[String]) = songsData.map(songData => { - val elements = songData.split(",") - val trackId = elements(0) - val mxmTrackId = elements(1) - val words_map = extractSongWords(elements.slice(2, elements.size)) - new Song(trackId, mxmTrackId, words_map); - }) - - private def extractSongWords(words: Array[String]): Map[Int, Int] = words.toSeq.map(source_pair => { - val pair = source_pair.split(":") - assert(pair.size == 2) - pair(0).toInt -> pair(1).toInt - }).toMap; -} - - diff --git a/raw_data_processor/src/main/scala/scalalab3/lyricsengine/repositories/MillionSongDataSubSetRepository.scala b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/repositories/MillionSongDataSubSetRepository.scala new file mode 100644 index 0000000..f83aadb --- /dev/null +++ b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/repositories/MillionSongDataSubSetRepository.scala @@ -0,0 +1,18 @@ +package scalalab3.lyricsengine.repositories + + +import scalalab3.lyricsengine.system.{ProjectConfig, ProjectLogger} + +/** + * Repository for mongoDB + */ +// TODO: move to API? +class MillionSongDataSubSetRepository extends ProjectConfig with ProjectLogger { + def saveSongs(songs: Any) = ??? + + def saveWords(songs: Any) = ??? + + + def findAll() = ??? +} + diff --git a/raw_data_processor/src/main/scala/scalalab3/lyricsengine/system/ProjectConfig.scala b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/system/ProjectConfig.scala new file mode 100644 index 0000000..e6dc7bd --- /dev/null +++ b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/system/ProjectConfig.scala @@ -0,0 +1,24 @@ +package scalalab3.lyricsengine.system + +import com.typesafe.config.ConfigFactory + + +trait ProjectConfig { + val conf = ConfigFactory.load() + // Config variable: + // ---- Million Song Dataset data: + val PATH_TO_MSD_SUBSET_VAR_NAME = "msd.pathToMsd" + // ---- MongoDB: + val MONGODB_HOST_VAR_NAME = "mongoDb.host" + val MONGODB_PORT_VAR_NAME = "mongoDb.port" + val MONGODB_DB_NAME_VAR_NAME = "mongoDb.dbName" + + + def getPathToData() = conf.getString(PATH_TO_MSD_SUBSET_VAR_NAME) + + def getMongoDbHost() = conf.getString(MONGODB_HOST_VAR_NAME) + + def getMongoDbPort() = conf.getInt(MONGODB_PORT_VAR_NAME) + + def getMongoDbName() = conf.getString(MONGODB_DB_NAME_VAR_NAME) +} diff --git a/raw_data_processor/src/main/scala/scalalab3/lyricsengine/system/ProjectLogger.scala b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/system/ProjectLogger.scala new file mode 100644 index 0000000..dc5c72c --- /dev/null +++ b/raw_data_processor/src/main/scala/scalalab3/lyricsengine/system/ProjectLogger.scala @@ -0,0 +1,8 @@ +package scalalab3.lyricsengine.system + +import com.typesafe.scalalogging.Logger +import org.slf4j.LoggerFactory + +trait ProjectLogger { + val logger = Logger(LoggerFactory.getLogger("common")) +} diff --git a/raw_data_processor/src/test/resources/test_msd.txt b/raw_data_processor/src/test/resources/test_msd.txt index 640ea37..e04a3dd 100644 --- a/raw_data_processor/src/test/resources/test_msd.txt +++ b/raw_data_processor/src/test/resources/test_msd.txt @@ -15,6 +15,6 @@ # starts at 1 (not zero!) # All our work is done using UTF-8 encoding. # enjoy! -%i,the,you,to,and,a,me,it,not,in,my,is,of,your,that,do,on,are,we,am,will,all,for,no,be,have,love,so,know,this,but,with,what,just,when,like,now,que,time,can,come,de,there,go,up,oh,la,one,they,out,down,get,she,was,see,if,got,never,from,he,feel,want,let,make,way,say,take,would,as,ca,day,at,babi,away -TRZZZYV128F92E996D,6849828,1:10,2:6,3:20,5:2,7:30,8:1,9:6,10:3,11:19,12:1,13:16,14:6,15:1,18:4,20:2,21:2,23:1,24:4,26:4,31:3,35:1,37:2,50:15,52:21,56:1,57:3,59:3,63:6,68:2,74:3,75:3,84:1,92:1,94:3,100:1,101:2,111:2,121:3,123:5,142:3,144:1,155:1,162:31,167:6,184:3,203:3,216:1,227:1,240:2,261:1,358:1,368:2,372:4,382:3,403:2,415:2,434:1,467:3,485:1,571:1,592:3,627:4,672:6,1023:1,1123:1,1147:1,1247:1,1262:1,1294:1,1375:2,1391:15,1542:1,1619:1,2386:1,4817:3 -TRZZZYX128F92D32C6,681124,1:4,2:18,4:3,5:6,6:9,7:1,8:5,9:1,10:12,11:2,12:9,13:5,15:4,17:2,18:4,19:4,21:2,22:3,25:3,29:2,30:3,32:5,34:3,35:2,37:2,39:3,43:2,44:7,45:2,49:2,50:2,52:8,54:2,56:1,59:1,63:10,64:1,66:1,71:2,72:1,78:1,79:1,84:2,91:1,92:2,95:1,96:1,103:1,110:1,115:1,127:1,134:4,135:2,136:3,137:1,138:2,140:1,146:2,150:1,152:1,192:1,206:1,207:1,222:3,239:1,248:2,258:1,270:1,274:1,283:1,294:2,300:1,305:1,318:1,337:1,338:1,346:1,347:1,349:1,371:1,398:1,406:1,445:1,451:1,459:1,478:1,487:1,492:1,502:3,516:1,539:1,548:2,549:1,553:1,592:2,617:1,633:1,666:1,681:1,727:1,774:1,775:7,789:1,811:1,844:1,915:1,942:1,950:1,979:1,1008:1,1040:1,1080:9,1142:1,1232:1,1366:1,1409:1,1412:1,1537:1,1545:1,1597:1,1841:2,1861:1,1901:1,1907:1,2063:1,2097:1,2167:1,2198:1,2221:1,2468:1,2498:1,2595:1,2698:1,2975:1,2990:1,2996:1,3256:1,3267:1,3316:2,3355:1,4198:1,4356:1,4738:1,4845:1 \ No newline at end of file +%i,the,you,to,and +TRZZZYV128F92E996D,6849828,1:10,2:6,3:20,5:2,7:30 +TRZZZYX128F92D32C6,681124,1:4,2:18,4:3,5:6,6:9 \ No newline at end of file diff --git a/raw_data_processor/src/test/scala/MillionSongDataSubSetReaderTest.scala b/raw_data_processor/src/test/scala/MillionSongDataSubSetReaderTest.scala new file mode 100644 index 0000000..ff1d300 --- /dev/null +++ b/raw_data_processor/src/test/scala/MillionSongDataSubSetReaderTest.scala @@ -0,0 +1,30 @@ +import org.scalatest.{FlatSpec, Matchers} + +import scalalab3.lyricsengine.reader.{MillionSongDataSubSet, MillionSongDataSubSetReader, Song} + +class MillionSongDataSubSetReaderTest extends FlatSpec with Matchers { + + "A MillionSongDataSubSetReader" should "correct load all data" in { + // --- Prepare expected data: + val expectedData: MillionSongDataSubSet = prepareTestData(); + + // --- Computation actual data: + val path = getClass.getResource("/test_msd.txt").getPath + val actualData: MillionSongDataSubSet = MillionSongDataSubSetReader.extract(path) + + // --- Checking data: + assert(expectedData.words == actualData.words) + assert(expectedData.songs.size == actualData.songs.size) + actualData.songs(0) == expectedData.songs(0) + actualData.songs(1) == expectedData.songs(1) + //Just in case. + assert(expectedData.songs == actualData.songs) + } + + def prepareTestData() = { + val words = List("i", "the", "you", "to", "and") + val firstSong = new Song("TRZZZYV128F92E996D", "6849828", Map(1 -> 10, 2 -> 6, 3 -> 20, 5 -> 2, 7 -> 30)) + val secondSong = new Song("TRZZZYX128F92D32C6", "681124", Map(1 -> 4, 2 -> 18, 4 -> 3, 5 -> 6, 6 -> 9)) + new MillionSongDataSubSet(words, Seq(firstSong, secondSong)) + } +} diff --git a/raw_data_processor/src/test/scala/ReaderTest.scala b/raw_data_processor/src/test/scala/ReaderTest.scala deleted file mode 100644 index 2d2d1d0..0000000 --- a/raw_data_processor/src/test/scala/ReaderTest.scala +++ /dev/null @@ -1,9 +0,0 @@ -import org.scalatest.{FlatSpec, Matchers} - -class ReaderTest extends FlatSpec with Matchers { - - "A MSD" should "contains" in { - // val raw = Source.fromInputStream(getClass.getResourceAsStream("/test_msd.txt")).mkString - // Reader.extract("/test_msd.txt") - } -}