diff --git a/build.sbt b/build.sbt index b8ff8f3..319336d 100644 --- a/build.sbt +++ b/build.sbt @@ -1,3 +1,9 @@ +import _root_.sbt.Keys._ + +name := "lyrics-engine" + +version := "1.0-SNAPSHOT" + name := "lyrics-engine" lazy val commonSettings = Seq( @@ -16,10 +22,17 @@ lazy val parser = (project in file("parser")) .dependsOn(domain) lazy val api = (project in file("api")) .settings(commonSettings: _*) + +lazy val persistence = (project in file("persistence")) + .settings(commonSettings: _*) + .dependsOn(domain) lazy val root = (project in file(".")) .settings(commonSettings: _*) .dependsOn(api) - .aggregate(domain, parser, api) + .aggregate(domain, parser, api, persistence) .enablePlugins(UniversalPlugin, JavaAppPackaging) + + + diff --git a/persistence/build.sbt b/persistence/build.sbt new file mode 100644 index 0000000..fa50396 --- /dev/null +++ b/persistence/build.sbt @@ -0,0 +1,24 @@ +import sbt.Keys._ + +name := "persistence" + +organization := "scalalab3.lyricsengine" + +version := "0.1.0-SNAPSHOT" + +scalaVersion := "2.11.8" + +resolvers += "releases" at "https://oss.sonatype.org/content/groups/scala-tools" + +val specsV = "3.7.2" +val scalaTestV = "2.2.6" + +libraryDependencies ++= + Seq( + "org.scalatest" %% "scalatest" % scalaTestV, + "org.specs2" %% "specs2-core" % specsV, + "org.specs2" %% "specs2-matcher-extra" % specsV, + "org.mongodb" %% "casbah" % "3.1.1", + "com.typesafe" % "config" % "1.3.0" + ) + diff --git a/persistence/src/main/resources/application.conf b/persistence/src/main/resources/application.conf new file mode 100644 index 0000000..e6cbba5 --- /dev/null +++ b/persistence/src/main/resources/application.conf @@ -0,0 +1,10 @@ +mongo { + host= "localhost" + port= "8080" + user="" + password="" + dbname= "lyrics" + songscollection= "songs" + wordsdefinitions= "wordsdefinitions" +} + diff --git a/persistence/src/main/scala/scalalab3/lyricsengine/persistence/StorageComponent.scala b/persistence/src/main/scala/scalalab3/lyricsengine/persistence/StorageComponent.scala new file mode 100644 index 0000000..2f8ac9a --- /dev/null +++ b/persistence/src/main/scala/scalalab3/lyricsengine/persistence/StorageComponent.scala @@ -0,0 +1,20 @@ +package scalalab3.lyricsengine.persistence + +import scalalab3.lyricsengine.domain._ + +/** + * Created by annie on 5/17/16. + */ +trait StorageComponent { + + val storage: Storage + + trait Storage { + def addDataSet(dataSet: DataSet, version: Option[Int] = None) + + def getDataSet(version: Option[Int] = None): DataSet + + def getLastVersion: Int + } + +} diff --git a/persistence/src/main/scala/scalalab3/lyricsengine/persistence/StorageComponentImpl.scala b/persistence/src/main/scala/scalalab3/lyricsengine/persistence/StorageComponentImpl.scala new file mode 100644 index 0000000..65e0687 --- /dev/null +++ b/persistence/src/main/scala/scalalab3/lyricsengine/persistence/StorageComponentImpl.scala @@ -0,0 +1,115 @@ +package scalalab3.lyricsengine.persistence + +import com.mongodb.casbah.Imports._ + +import scalalab3.lyricsengine.domain._ +import scalalab3.lyricsengine.persistence.mongo.MongoContext + +trait StorageComponentImpl extends StorageComponent { + + override val storage: Storage + + class StorageImpl(implicit context: MongoContext) extends Storage { + + def addDataSet(dataSet: DataSet, version: Option[Int] = None) = { + addWordsDefinition(dataSet.definition, version) + addSongs(dataSet.songs, version) + } + + def getDataSet(version: Option[Int] = None): DataSet = { + val wd = findWordsDefinitions(version) + val songs = findSongs(version) + DataSet(wd, songs) + } + + def getLastVersion: Int = { + val query = MongoDBObject() // All documents + val fields = MongoDBObject("version" -> 1) // Only return `version` + val orderBy = MongoDBObject("version" -> -1) // Order by version descending + val result = context.wdCollection.findOne(query, fields, orderBy) + val defaultVersion = -1 + result match { + case Some(res) => res.getAs[Int]("version").getOrElse(defaultVersion) + case None => defaultVersion + } + } + + private def addSongs(songsToAdd: Seq[Song], version: Option[Int] = None) = { + val builder = context.songsCollection.initializeOrderedBulkOperation //will automatically split the operation into batches + for { + song <- songsToAdd + } builder.insert(songToMongoDBObj(song, version)) + val result = builder.execute() + } + + private def addWordsDefinition(wd: WordsDefinition, version: Option[Int] = None) = { + val builder = context.wdCollection.initializeOrderedBulkOperation + builder.insert(wdToMongoDbObject(wd, version)) + val result = builder.execute() + } + + private def findSongs(version: Option[Int] = None): Seq[Song] = { + val query = MongoDBObject("version" -> version.getOrElse(getLastVersion)) + val result = context.songsCollection.find(query) + val songsSet = for { + song <- result + } yield songFromMongoDBObj(song) + songsSet.toSeq + } + + private def findWordsDefinitions(version: Option[Int] = None): WordsDefinition = { + val query = MongoDBObject("version" -> version.getOrElse(getLastVersion)) + val result = context.wdCollection.find(query) + result.map(wdFromMongoDBObj(_)).next() + } + + private def countSongs(): Int = context.songsCollection.find().count() + + private def countWD(): Int = context.wdCollection.find().count() + + private def songFromMongoDBObj(obj: MongoDBObject): Song = { + val _msdTrackId = obj.as[String]("msdTrackId") + val _mxmTrackId = obj.as[String]("mxmTrackId") + val _words = obj.as[Map[Int, Int]]("words") + Song(msdTrackId = _msdTrackId, mxmTrackId = _mxmTrackId, words = _words) + } + + private def wdFromMongoDBObj(obj: MongoDBObject): Map[Int, String] = { + obj.getAs[Map[String, String]]("wordsDefinitions").get.map { case (k, v) => (k.toInt, v.toString) } + } + + private def songToMongoDBObj(song: Song, version: Option[Int]): MongoDBObject = { + val songBuilder = MongoDBObject.newBuilder + songBuilder +=("msdTrackId" -> song.msdTrackId, + "mxmTrackId" -> song.mxmTrackId, + "words" -> getWDefinitions(song), + "version" -> version.getOrElse(getLastVersion) + ) + songBuilder.result() + } + + private def wdToMongoDbObject(wd: WordsDefinition, version: Option[Int]): MongoDBObject = { + val wdBuilder = MongoDBObject.newBuilder + wdBuilder +=( + "wordsDefinitions" -> transformWordsDef(wd), + "version" -> version.getOrElse(getLastVersion) + ) + wdBuilder.result() + } + + private def transformWordsDef(w: WordsDefinition): MongoDBObject = { + val collection = MongoDBObject.newBuilder + val transformed = w.map { case (k, v) => (k.toString, v) } + collection ++= transformed + collection.result() + } + + private def getWDefinitions(song: Song): MongoDBObject = { + val words = MongoDBObject.newBuilder + val songWords = song.words.map { case (k, v) => (k.toString, v) } + words ++= songWords + words.result() + } + } + +} \ No newline at end of file diff --git a/persistence/src/main/scala/scalalab3/lyricsengine/persistence/mongo/MongoConfig.scala b/persistence/src/main/scala/scalalab3/lyricsengine/persistence/mongo/MongoConfig.scala new file mode 100644 index 0000000..54000fe --- /dev/null +++ b/persistence/src/main/scala/scalalab3/lyricsengine/persistence/mongo/MongoConfig.scala @@ -0,0 +1,32 @@ +package scalalab3.lyricsengine.persistence.mongo + +import com.typesafe.config.ConfigFactory + +import constants.DefaultConfigValues._ +import constants.DefaultConfigKeys._ +import scala.util.Try + +case class MongoConfig(host: String = defaultHost, + port: Int = defaultPort, + user: String = defaultUser, + password: String = defaultPassword, + dbName: String = defaultDbName, + songsCollection: String = defaultSongsCollection, + wdCollection: String = defaultWDCollection) + +object MongoConfig { + private val config = ConfigFactory.load() + + def load(): MongoConfig = + MongoConfig( + getString(host, defaultHost), + getInt(port, defaultPort), + getString(user, defaultUser), + getString(password, defaultPassword), + getString(dbname, defaultDbName), + getString(songsCollection, defaultSongsCollection), + getString(wordsDefinitions, defaultWDCollection) + ) + private def getString(key: String, defaultValue: String) = Try(config.getString(key)).getOrElse(defaultValue) + private def getInt(key: String, defaultValue: Int) = Try(config.getInt(key)).getOrElse(defaultValue) +} diff --git a/persistence/src/main/scala/scalalab3/lyricsengine/persistence/mongo/MongoContext.scala b/persistence/src/main/scala/scalalab3/lyricsengine/persistence/mongo/MongoContext.scala new file mode 100644 index 0000000..938d723 --- /dev/null +++ b/persistence/src/main/scala/scalalab3/lyricsengine/persistence/mongo/MongoContext.scala @@ -0,0 +1,37 @@ +package scalalab3.lyricsengine.persistence.mongo + +import com.mongodb.BasicDBObjectBuilder +import com.mongodb.casbah.Imports._ +import com.mongodb.casbah.MongoClient + +class MongoContext(val config: MongoConfig) { + + val mongoClient = MongoClient() + val mongoDB = mongoClient(config.dbName) + + implicit val options: DBObject = BasicDBObjectBuilder.start().add("capped", true).add("size", 2000000000l).get() + + def songsCollection = + if (mongoDB.collectionExists(config.songsCollection)) { + mongoDB(config.songsCollection) + } else { + mongoDB.createCollection(config.songsCollection, options) + mongoDB(config.songsCollection) + } + + def wdCollection = { + if (mongoDB.collectionExists(config.wdCollection)) { + mongoDB(config.wdCollection) + } else { + mongoDB.createCollection(config.wdCollection, options) + mongoDB(config.wdCollection) + } + } + + def drop = { + mongoDB.dropDatabase() + } + + + +} diff --git a/persistence/src/main/scala/scalalab3/lyricsengine/persistence/mongo/constants/DefaultConfigKeys.scala b/persistence/src/main/scala/scalalab3/lyricsengine/persistence/mongo/constants/DefaultConfigKeys.scala new file mode 100644 index 0000000..166ea53 --- /dev/null +++ b/persistence/src/main/scala/scalalab3/lyricsengine/persistence/mongo/constants/DefaultConfigKeys.scala @@ -0,0 +1,13 @@ +package scalalab3.lyricsengine.persistence.mongo.constants + +object DefaultConfigKeys { + val mongo = "mongo" + val host = s"$mongo.host" + val port = s"$mongo.port" + val user = s"$mongo.user" + val dbname = s"$mongo.dbname" + val password = s"$mongo.password" + val songsCollection = s"$mongo.songscollection" + val wordsDefinitions = s"$mongo.wordsdefinitions" + +} diff --git a/persistence/src/main/scala/scalalab3/lyricsengine/persistence/mongo/constants/DefaultConfigValues.scala b/persistence/src/main/scala/scalalab3/lyricsengine/persistence/mongo/constants/DefaultConfigValues.scala new file mode 100644 index 0000000..ce27c88 --- /dev/null +++ b/persistence/src/main/scala/scalalab3/lyricsengine/persistence/mongo/constants/DefaultConfigValues.scala @@ -0,0 +1,11 @@ +package scalalab3.lyricsengine.persistence.mongo.constants + +object DefaultConfigValues { + val defaultHost = "localhost" + val defaultPort = 27017 + val defaultUser = "admin" + val defaultPassword = "" + val defaultDbName = "lyrics" + val defaultSongsCollection = "songs" + val defaultWDCollection = "songs" +} diff --git a/persistence/src/test/scala/scalalab3/lyricsengine/persistence/StorageComponentImpl$Test.scala b/persistence/src/test/scala/scalalab3/lyricsengine/persistence/StorageComponentImpl$Test.scala new file mode 100644 index 0000000..82bd63f --- /dev/null +++ b/persistence/src/test/scala/scalalab3/lyricsengine/persistence/StorageComponentImpl$Test.scala @@ -0,0 +1,56 @@ +package scalalab3.lyricsengine.persistence + +import java.util.UUID +import org.specs2.mutable.Specification +import org.specs2.specification._ +import scala.util.Try +import scalalab3.lyricsengine.domain.{DataSet, Song} +import scalalab3.lyricsengine.persistence.mongo.{MongoConfig, MongoContext} + +class StorageComponentImpl$Test extends Specification with BeforeAfterAll { + sequential + + val tryMongoContext = Try(new MongoContext(MongoConfig.load())) + + "MongoDB Test" >> { + if (tryMongoContext.isSuccess) { + implicit val s = tryMongoContext.get + val wd = Map(1 -> "i", 2 -> "the", 3 -> "you", 4 -> "to", 5 -> "and") + val firstSong = Song("TRZZZYV128F92E996D", "6849828", Map(1 -> 10, 2 -> 6, 3 -> 20, 5 -> 2, 7 -> 30)) + val secondSong = Song("TRZZZYX128F92D32C6", "681124", Map(1 -> 4, 2 -> 18, 4 -> 3, 5 -> 6, 6 -> 9)) + val seqSong = Seq(firstSong, secondSong) + val MdataSet=DataSet(wd, seqSong) + val mongoStorage = new StorageComponentImpl { + override val storage: Storage = new StorageImpl + }.storage + + "add DataSet " in { + mongoStorage.addDataSet(MdataSet) + mongoStorage.getDataSet().songs.size must_== seqSong.size + mongoStorage.getDataSet().definition.size must_== wd.size + } + + "get default LastVersion" in { + mongoStorage.getLastVersion must_== -1 + } + + "add DataSet with version " in { + mongoStorage.addDataSet(MdataSet, Some(1)) + mongoStorage.getDataSet().songs.size must_== seqSong.size + mongoStorage.getDataSet().definition.size must_== wd.size + } + + "get LastVersion" in { + mongoStorage.getLastVersion must_== 1 + } + + } else "Skipped Test" >> skipped("Mongo context is not available in ") + } + + def uuid() = Some(UUID.randomUUID()) + + def drop() = for (m <- tryMongoContext) m.drop + + override def beforeAll(): Unit = drop() + override def afterAll(): Unit = drop() +}