From 5b8cc73b5770674730a2884d2e2670ebde3fb738 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Fri, 1 Jun 2018 08:25:33 +0900 Subject: [PATCH 01/30] Update: version to 0.2.0b2-SNAPSHOT --- build.sbt | 2 +- src/main/resources/application.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 56e869c..bc23928 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ name := "backlog-migration-cybozulive" lazy val commonSettings = Seq( - version := "0.2.0b1", + version := "0.2.0b2-SNAPSHOT", scalaVersion := "2.12.6" ) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index c21f781..7ef9350 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -1,7 +1,7 @@ application { name = "Backlog Migration for CybozuLive" - version = "0.2.0b1" + version = "0.2.0b2-SNAPSHOT" title = ${application.name} ${application.version} (c) nulab.inc fileName = backlog-migration-cybozulive-${application.version}.jar language = default From 63abb7cf0156571e69e500ca25ccabd6af13e502 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Fri, 1 Jun 2018 10:39:05 +0900 Subject: [PATCH 02/30] Update: submodule. BLGMIGRATION-720 --- modules/common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/common b/modules/common index 8390ca6..23c4879 160000 --- a/modules/common +++ b/modules/common @@ -1 +1 @@ -Subproject commit 8390ca6461ccb680bcf817e71bfa75c119c7f3e7 +Subproject commit 23c48795f42ebd83f535e576579e782487d1ec2f From 8e34de4d7a725df2636445b0ff866273ae49577c Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Tue, 12 Jun 2018 14:52:29 +0900 Subject: [PATCH 03/30] Remove: limitation of Backlog. BLGMIGRATION-733 --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index a82e8c7..b62251b 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,6 @@ Backlogの **管理者権限** が必要になります。 - ToDoのカテゴリは移行できません。 ### Backlog側の制限について -* Backlogで登録可能なユーザー数を超えた場合、インポートは中断されます。 * 空のコメントは登録されません。 ## 再インポートの仕様 @@ -262,7 +261,6 @@ This program is for the users with the Space's **administrator** roles. - The ToDo category can not be migrated. ### About limitations in the Backlog -- Importing users will be terminated if the number of users exceeds the limit in the Backlog. - Empty comments are not registered. ## Re-importing From 6b574cad11e10019cc560f244472dd0253686add Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Thu, 14 Jun 2018 14:36:01 +0900 Subject: [PATCH 04/30] Add: notice. BLGMIGRATION-741 --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index b62251b..b997440 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,9 @@ Backlog側に同一プロジェクトキーがある場合は、以下の仕様 MacOSにおいて、`テキストエディット`アプリでマッピングファイルを編集するとダブルクオーテーションが変換されてしまいます。 「環境設定」→「スマート引用符」のチェックを外してください。 +### CybozuLiveのユーザー名 +ユーザー名の前後に空白が含まれていると移行できません。 + ## ライセンス MIT License @@ -283,6 +286,9 @@ To migrate this project, you have to join. Join the project to add issues. In MacOS, when you edit the mapping file with `TextEdit` application, double quotes will be converted. Please uncheck "Preferences" → "Smart quotes". +### CybozuLive user name +Can not migrate if the user name contains spaces before and after it. + ## License MIT License From 87facbaca1d725e19f9928be1b742ec06452e958 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Thu, 28 Jun 2018 13:11:55 +0900 Subject: [PATCH 05/30] Update: version to 1.0.1-SNAPSHOT --- build.sbt | 2 +- src/main/resources/application.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index feb39ac..815ef20 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ name := "backlog-migration-cybozulive" lazy val commonSettings = Seq( - version := "1.0.0", + version := "1.0.1-SNAPSHOT", scalaVersion := "2.12.6" ) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 57a1496..0f7673b 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -1,7 +1,7 @@ application { name = "Backlog Migration for CybozuLive" - version = "1.0.0" + version = "1.0.1-SNAPSHOT" title = ${application.name} ${application.version} (c) nulab.inc fileName = backlog-migration-cybozulive-${application.version}.jar language = default From 669231cfed1af406a951ea362559ffed67c4e33c Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Sun, 19 Aug 2018 11:48:51 +0900 Subject: [PATCH 06/30] Update: library version --- build.sbt | 4 ++-- project/build.properties | 2 +- project/plugins.sbt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 815ef20..10d5cba 100644 --- a/build.sbt +++ b/build.sbt @@ -24,7 +24,7 @@ lazy val root = (project in file(".")) .settings(commonSettings) .settings( libraryDependencies ++= { - val catsVersion = "1.1.0" + val catsVersion = "1.2.0" val slickVersion = "3.2.3" val monixVersion = "3.0.0-RC1" Seq( @@ -33,7 +33,7 @@ lazy val root = (project in file(".")) "org.typelevel" %% "cats-free" % catsVersion, "com.typesafe.slick" %% "slick" % slickVersion, "com.typesafe.slick" %% "slick-hikaricp" % slickVersion, - "org.xerial" % "sqlite-jdbc" % "3.21.0", + "org.xerial" % "sqlite-jdbc" % "3.23.1", "io.monix" %% "monix" % monixVersion, "io.monix" %% "monix-reactive" % monixVersion, "io.monix" %% "monix-execution" % monixVersion, diff --git a/project/build.properties b/project/build.properties index c3e3abc..8db5ca2 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 1.1.4 \ No newline at end of file +sbt.version = 1.2.1 \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt index e961a9c..aba1e48 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,3 @@ logLevel := Level.Warn -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6") \ No newline at end of file +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.7") \ No newline at end of file From ad5e505667c2c34f4800033b98a92da87d2d814f Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Sun, 19 Aug 2018 13:50:38 +0900 Subject: [PATCH 07/30] Add: ZonedDateTime parser for other language --- .../backlog/c2b/parsers/ZonedDateTimeParser.scala | 13 ++++++++++++- .../c2b/parsers/ZonedDateTimeParserSpec.scala | 6 ++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParser.scala b/src/main/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParser.scala index 40b70ec..15e39a6 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParser.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParser.scala @@ -1,11 +1,16 @@ package com.nulabinc.backlog.c2b.parsers +import java.text.SimpleDateFormat import java.time.{ZoneId, ZonedDateTime} import com.nulabinc.backlog.c2b.datas.Types.DateTime +import scala.util.{Failure, Success, Try} + object ZonedDateTimeParser { + private val otherDateFormat = new SimpleDateFormat("EEE, MMM dd, yyyy HH:mm") + def toZonedDateTime(value: String): Either[ParseError[DateTime], DateTime] = { val pattern1 = """(\d+?)/(\d+?)/(\d+?) .*?(\d+?):(\d+?):(\d+)""".r val pattern2 = """(\d+?)/(\d+?)/(\d+?) .*?(\d+?):(\d+?)""".r @@ -50,7 +55,13 @@ object ZonedDateTimeParser { ZoneId.systemDefault() ) ) - case _ => Left(CannotParseFromString(classOf[DateTime], value)) + case _ => + Try(otherDateFormat.parse(value)) match { + case Success(date) => + Right(ZonedDateTime.ofInstant(date.toInstant, ZoneId.systemDefault())) + case Failure(_) => + Left(CannotParseFromString(classOf[DateTime], value)) + } } } diff --git a/src/test/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParserSpec.scala b/src/test/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParserSpec.scala index 4d4fb36..74f7fb0 100644 --- a/src/test/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParserSpec.scala +++ b/src/test/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParserSpec.scala @@ -37,4 +37,10 @@ class ZonedDateTimeParserSpec extends FlatSpec with Matchers { val actual = ZonedDateTimeParser.toZonedDateTime(date, time) actual shouldEqual Right(ZonedDateTime.of(2015, 10, 5, 8, 23, 0, 0, ZoneId.systemDefault())) } + + "ZonedDateTimeParser" should "parse other formatted date time string" in { + val date = "Fri, Jul 13, 2018 8:36" + val actual = ZonedDateTimeParser.toZonedDateTime(date) + actual shouldEqual Right(ZonedDateTime.of(2018, 7, 13, 8, 36, 0, 0, ZoneId.systemDefault())) + } } From 70fb266b77f3092e0124d90b55c741e1c13b59b1 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Sun, 19 Aug 2018 15:02:27 +0900 Subject: [PATCH 08/30] Add: models --- .../backlog/c2b/datas/CybozuTextModels.scala | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuTextModels.scala diff --git a/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuTextModels.scala b/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuTextModels.scala new file mode 100644 index 0000000..e11bd98 --- /dev/null +++ b/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuTextModels.scala @@ -0,0 +1,17 @@ +package com.nulabinc.backlog.c2b.datas + +import com.nulabinc.backlog.c2b.datas.Types.DateTime + +case class CybozuTextTopic( + title: String, + description: String, + posts: Seq[CybozuTextPost] +) + +case class CybozuTextPost( + content: String, + postUser: CybozuTextUser, + postedAt: DateTime +) + +case class CybozuTextUser(value: String) extends AnyVal \ No newline at end of file From c68dc91ab07495aa4c9c5ef5c308e0e22a8f61aa Mon Sep 17 00:00:00 2001 From: Rikiya Kawakami Date: Tue, 21 Aug 2018 12:58:17 +0900 Subject: [PATCH 09/30] Remove beta version note from README #1 --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index b997440..e18bd24 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,6 @@ CybozuLiveのグループを[Backlog]に移行するためのツールです。 (English document is described after Japanese) -**Backlog Migration for CybozuLiveはベータバージョンです。Backlog上の既存プロジェクトにインポートする場合は、先に新しく別プロジェクトを作成し、こちらにインポートし内容を確認後、正式なプロジェクトにインポートしてください** - * Backlog * [https://backlog.com](https://backlog.com/) From 0a72a5b32f99e2a96856441598b4ddb43630acf6 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Tue, 21 Aug 2018 14:20:32 +0900 Subject: [PATCH 10/30] Add: text file to object --- .../c2b/readers/CybozuTextReader.scala | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 src/main/scala/com/nulabinc/backlog/c2b/readers/CybozuTextReader.scala diff --git a/src/main/scala/com/nulabinc/backlog/c2b/readers/CybozuTextReader.scala b/src/main/scala/com/nulabinc/backlog/c2b/readers/CybozuTextReader.scala new file mode 100644 index 0000000..654ea70 --- /dev/null +++ b/src/main/scala/com/nulabinc/backlog/c2b/readers/CybozuTextReader.scala @@ -0,0 +1,43 @@ +package com.nulabinc.backlog.c2b.readers + +import java.io.File +import java.nio.charset.Charset + +import monix.reactive.Observable + +import scala.io.Source + + +case class TopicReadResult(topicText: String, comments: Stream[String]) + +object CybozuTopicTextReader { + + private val charset = Charset.forName("UTF-8") + private val topicSeparator = "================================================================================" + private val commentSeparator = "--------------------------------------------------------------------------------" + + private case class Last[T](topicText: String, comments: Stream[T], last: T) + + def read(files: Array[File]): Observable[TopicReadResult] = + Observable + .fromIterable(files) + .flatMap { file => + Observable + .fromIterator(Source.fromFile(file.getAbsolutePath, charset.name).getLines) + .drop(1) + .foldLeftF(Last("", Stream.empty[String], "")) { + case (acc, line) => + if (line.startsWith(topicSeparator) || line.startsWith(commentSeparator)) { + if (acc.topicText.isEmpty) + Last(acc.last, acc.comments, "") + else + Last(acc.topicText, acc.comments :+ acc.last, "") + } else { + Last(acc.topicText, acc.comments, acc.last + line) + } + }.map { result => + TopicReadResult(result.topicText, result.comments) + } + } + +} From 1363fcfe98681942de853dd6897bda210a2fed4c Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Wed, 22 Aug 2018 15:29:02 +0900 Subject: [PATCH 11/30] Add: load text files --- src/main/resources/messages.txt | 1 + src/main/resources/messages_ja.txt | 1 + .../backlog/c2b/services/CybozuStore.scala | 21 ++++++++++++------- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/main/resources/messages.txt b/src/main/resources/messages.txt index f198008..747c1f0 100644 --- a/src/main/resources/messages.txt +++ b/src/main/resources/messages.txt @@ -13,6 +13,7 @@ name.mapping.user=User name.mapping.priority=Priority name.mapping.status=Status name.status.open=Open +name.chat=Chat # Validation validation.access=Checking whether the {0} is accessible ... diff --git a/src/main/resources/messages_ja.txt b/src/main/resources/messages_ja.txt index dac451b..1682117 100644 --- a/src/main/resources/messages_ja.txt +++ b/src/main/resources/messages_ja.txt @@ -13,6 +13,7 @@ name.mapping.user=ユーザー name.mapping.priority=優先度 name.mapping.status=状態 name.status.open=未完了 +name.chat=チャット # Validation validation.access={0}にアクセス可能かチェックしています... diff --git a/src/main/scala/com/nulabinc/backlog/c2b/services/CybozuStore.scala b/src/main/scala/com/nulabinc/backlog/c2b/services/CybozuStore.scala index 72ce8bc..692a86b 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/services/CybozuStore.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/services/CybozuStore.scala @@ -14,24 +14,26 @@ import com.osinka.i18n.Messages object CybozuStore extends Logger { - def copyToStore(csvFiles: Array[File]): AppProgram[Unit] = { + def copyToStore(exportedFiles: Array[File]): AppProgram[Unit] = { val todoFiles = { - csvFiles.filter(_.getName.contains("live_ToDoリスト_")) ++ - csvFiles.filter(_.getName.contains("live_To-Do List_")) + exportedFiles.filter(_.getName.contains("live_ToDoリスト_")) ++ + exportedFiles.filter(_.getName.contains("live_To-Do List_")) } val eventFiles = { - csvFiles.filter(_.getName.contains("live_イベント_")) ++ - csvFiles.filter(_.getName.contains("live_Events_")) + exportedFiles.filter(_.getName.contains("live_イベント_")) ++ + exportedFiles.filter(_.getName.contains("live_Events_")) } val forumFiles = { - csvFiles.filter(_.getName.contains("live_掲示板_")) ++ - csvFiles.filter(_.getName.contains("live_Forum_")) + exportedFiles.filter(_.getName.contains("live_掲示板_")) ++ + exportedFiles.filter(_.getName.contains("live_Forum_")) } + val chatFiles = exportedFiles.filter(_.getName.endsWith(".txt")) for { _ <- todo(todoFiles) _ <- event(eventFiles) _ <- forum(forumFiles) + _ <- chat(chatFiles) } yield () } @@ -121,6 +123,11 @@ object CybozuStore extends Logger { ) } yield () + def chat(files: Array[File]): AppProgram[Unit] = + for { + _ <- AppDSL.fromConsole(ConsoleDSL.print(Messages("message.init.collect", Messages("name.chat")))) + } yield () + private def sequential[A](prgs: Seq[StoreProgram[A]]): StoreProgram[Seq[A]] = prgs.foldLeft(StoreDSL.pure(Seq.empty[A])) { case (newPrg, prg) => From 0c447b415fbffe4e8fc7c269ddd5b34fff3c3de1 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Thu, 23 Aug 2018 10:31:50 +0900 Subject: [PATCH 12/30] Add: table definition --- .../backlog/c2b/datas/CybozuDBModels.scala | 18 ++++++++ .../backlog/c2b/datas/CybozuModels.scala | 7 +++ .../backlog/c2b/datas/CybozuTextModels.scala | 1 - .../interpreters/sqlite/ops/AllTableOps.scala | 4 ++ .../sqlite/ops/ChatTableOps.scala | 43 +++++++++++++++++++ .../sqlite/ops/CommentTableOps.scala | 4 ++ .../sqlite/tables/ChatTable.scala | 16 +++++++ .../sqlite/tables/CommentTable.scala | 3 +- 8 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/ChatTableOps.scala create mode 100644 src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/tables/ChatTable.scala diff --git a/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuDBModels.scala b/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuDBModels.scala index bc2d851..ce964ab 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuDBModels.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuDBModels.scala @@ -115,4 +115,22 @@ object CybozuDBForum { updater = updaterId, updatedAt = forum.updatedAt ) +} + +case class CybozuDBChat( + id: AnyId, + title: String, + description: String +) extends Entity + +object CybozuDBChat { + + val tupled = (this.apply _).tupled + + def from(topic: CybozuTextTopic): CybozuDBChat = + CybozuDBChat( + id = 0, + title = topic.title, + description = topic.description + ) } \ No newline at end of file diff --git a/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuModels.scala b/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuModels.scala index e4a47ce..011deab 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuModels.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuModels.scala @@ -50,6 +50,13 @@ case class CybozuForum( comments: Seq[CybozuComment] ) +case class CybozuChat( + id: AnyId, + title: String, + description: String, + comments: Seq[CybozuComment] +) + case class CybozuComment( id: AnyId, parentId: AnyId, diff --git a/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuTextModels.scala b/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuTextModels.scala index e11bd98..d8a7530 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuTextModels.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuTextModels.scala @@ -5,7 +5,6 @@ import com.nulabinc.backlog.c2b.datas.Types.DateTime case class CybozuTextTopic( title: String, description: String, - posts: Seq[CybozuTextPost] ) case class CybozuTextPost( diff --git a/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/AllTableOps.scala b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/AllTableOps.scala index 38d9189..8eac3fa 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/AllTableOps.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/AllTableOps.scala @@ -7,9 +7,11 @@ private[sqlite] case class AllTableOps()(implicit exc: Scheduler) { val todoCommentTableOps = ToDoCommentTableOps() val eventCommentTableOps = EventCommentTableOps() val forumCommentTableOps = ForumCommentTableOps() + val chatCommentTableOps = ChatCommentTableOps() val eventTableOps = EventTableOps() val forumTableOps = ForumTableOps() val todoTableOps = TodoTableOps() + val chatTableOps = ChatTableOps() val backlogUserTableOps = BacklogUserTableOps() val backlogPriorityTableOps = BacklogPriorityTableOps() val backlogStatusTableOps = BacklogStatusTableOps() @@ -24,6 +26,8 @@ private[sqlite] case class AllTableOps()(implicit exc: Scheduler) { eventCommentTableOps.createTable, forumTableOps.createTable, forumCommentTableOps.createTable, + chatTableOps.createTable, + chatCommentTableOps.createTable, backlogUserTableOps.createTable, backlogPriorityTableOps.createTable, backlogStatusTableOps.createTable, diff --git a/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/ChatTableOps.scala b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/ChatTableOps.scala new file mode 100644 index 0000000..6ecd66e --- /dev/null +++ b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/ChatTableOps.scala @@ -0,0 +1,43 @@ +package com.nulabinc.backlog.c2b.persistence.interpreters.sqlite.ops + +import com.nulabinc.backlog.c2b.datas.{CybozuChat, CybozuDBChat, CybozuDBComment} +import com.nulabinc.backlog.c2b.datas.Types.AnyId +import com.nulabinc.backlog.c2b.persistence.interpreters.sqlite.core.DBIOTypes.DBIORead +import com.nulabinc.backlog.c2b.persistence.interpreters.sqlite.tables.{ChatCommentTable, ChatTable, CybozuUserTable} +import slick.lifted.TableQuery +import slick.jdbc.SQLiteProfile.api._ + +import scala.concurrent.ExecutionContext + +private[sqlite] case class ChatTableOps()(implicit exec: ExecutionContext) extends BaseTableOps[CybozuDBChat, ChatTable] { + + protected val tableQuery = TableQuery[ChatTable] + private val commentTableQuery = TableQuery[ChatCommentTable] + private val cybozuUserTableQuery = TableQuery[CybozuUserTable] + + def getChat(id: AnyId): DBIORead[Option[CybozuChat]] = + for { + optChat <- tableQuery + .filter(_.id === id) + .result + .headOption + comments <- commentTableQuery + .filter(_.parentId === id) + .join(cybozuUserTableQuery) + .on(_.creator === _.id) + .sortBy(_._1.id.desc) + .result + } yield { + optChat.map(chat => + CybozuChat( + id = chat.id, + title = chat.title, + description = chat.description, + comments = comments.map { + case (comment, commentCreator) => + CybozuDBComment.to(comment, commentCreator) + } + ) + ) + } +} diff --git a/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/CommentTableOps.scala b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/CommentTableOps.scala index 13d6483..4aa6a24 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/CommentTableOps.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/CommentTableOps.scala @@ -27,3 +27,7 @@ private[sqlite] case class EventCommentTableOps()(implicit exc: Scheduler) exten private[sqlite] case class ForumCommentTableOps()(implicit exc: Scheduler) extends BaseTableOps[CybozuDBComment, ForumCommentTable] { override val tableQuery = TableQuery[ForumCommentTable] } + +private[sqlite] case class ChatCommentTableOps()(implicit exc: Scheduler) extends BaseTableOps[CybozuDBComment, ChatCommentTable] { + override val tableQuery = TableQuery[ChatCommentTable] +} \ No newline at end of file diff --git a/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/tables/ChatTable.scala b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/tables/ChatTable.scala new file mode 100644 index 0000000..6a9fffb --- /dev/null +++ b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/tables/ChatTable.scala @@ -0,0 +1,16 @@ +package com.nulabinc.backlog.c2b.persistence.interpreters.sqlite.tables + +import com.nulabinc.backlog.c2b.datas.CybozuDBChat +import slick.lifted.{ProvenShape, Tag} +import slick.jdbc.SQLiteProfile.api._ + + +private[sqlite] class ChatTable(tag: Tag) extends BaseTable[CybozuDBChat](tag, "cybozu_chats") { + + def title: Rep[String] = column[String]("title") + def description: Rep[String] = column[String]("description") + + override def * : ProvenShape[CybozuDBChat] = + (id, title, description) <> (CybozuDBChat.tupled, CybozuDBChat.unapply) + +} \ No newline at end of file diff --git a/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/tables/CommentTable.scala b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/tables/CommentTable.scala index 1954f26..11a06d2 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/tables/CommentTable.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/tables/CommentTable.scala @@ -20,4 +20,5 @@ private[sqlite] abstract class CommentTable(tag: Tag, name: String) extends Base private[sqlite] class ToDoCommentTable(tag: Tag) extends CommentTable(tag, "cybozu_todo_comments") private[sqlite] class EventCommentTable(tag: Tag) extends CommentTable(tag, "cybozu_event_comments") -private[sqlite] class ForumCommentTable(tag: Tag) extends CommentTable(tag, "cybozu_forum_comments") \ No newline at end of file +private[sqlite] class ForumCommentTable(tag: Tag) extends CommentTable(tag, "cybozu_forum_comments") +private[sqlite] class ChatCommentTable(tag: Tag) extends CommentTable(tag, "cybozu_chat_comments") \ No newline at end of file From 4049a7b43e14127abe331a93bb46bf0ae2482d42 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Thu, 23 Aug 2018 11:58:26 +0900 Subject: [PATCH 13/30] Add: chat algebra --- .../c2b/persistence/dsl/StoreADT.scala | 6 ++++++ .../c2b/persistence/dsl/StoreDSL.scala | 15 ++++++++++++++ .../interpreters/StoreInterpreter.scala | 12 +++++++++++ .../sqlite/SQLiteInterpreter.scala | 20 +++++++++++++++++++ .../sqlite/ops/ChatTableOps.scala | 5 +++++ 5 files changed, 58 insertions(+) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/persistence/dsl/StoreADT.scala b/src/main/scala/com/nulabinc/backlog/c2b/persistence/dsl/StoreADT.scala index ed27448..2d0a9e3 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/persistence/dsl/StoreADT.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/persistence/dsl/StoreADT.scala @@ -15,6 +15,7 @@ sealed trait CommentType case object TodoComment extends CommentType case object EventComment extends CommentType case object ForumComment extends CommentType +case object ChatComment extends CommentType case class Pure[A](a: A) extends StoreADT[A] @@ -40,6 +41,11 @@ case object GetForumCount extends StoreADT[Int] case class GetForum(id: AnyId) extends StoreADT[Option[CybozuForum]] case class StoreForum(forum: CybozuDBForum, writeType: WriteType = Insert) extends StoreADT[AnyId] +case object GetChats extends StoreADT[Observable[CybozuDBChat]] +case object GetChatCount extends StoreADT[Int] +case class GetChat(id: AnyId) extends StoreADT[Option[CybozuChat]] +case class StoreChat(chat: CybozuDBChat, writeType: WriteType = Insert) extends StoreADT[AnyId] + case object GetCybozuUsers extends StoreADT[Observable[CybozuUser]] case class GetCybozuUserById(id: AnyId) extends StoreADT[Option[CybozuUser]] case class GetCybozuUserBykey(key: String) extends StoreADT[Option[CybozuUser]] diff --git a/src/main/scala/com/nulabinc/backlog/c2b/persistence/dsl/StoreDSL.scala b/src/main/scala/com/nulabinc/backlog/c2b/persistence/dsl/StoreDSL.scala index 28c4a29..a1aa471 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/persistence/dsl/StoreDSL.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/persistence/dsl/StoreDSL.scala @@ -87,6 +87,21 @@ object StoreDSL { def storeForum(forum: CybozuDBForum): StoreProgram[AnyId] = Free.liftF(StoreForum(forum)) + // + // Cybozu Chat + // + lazy val getChats: StoreProgram[Observable[CybozuDBChat]] = + Free.liftF(GetChats) + + lazy val getChatCount: StoreProgram[Int] = + Free.liftF(GetChatCount) + + def getChat(id: AnyId): StoreProgram[Option[CybozuChat]] = + Free.liftF(GetChat(id)) + + def storeChat(chat: CybozuDBChat): StoreProgram[AnyId] = + Free.liftF(StoreChat(chat)) + // // Cybozu user // diff --git a/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/StoreInterpreter.scala b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/StoreInterpreter.scala index ec63ea7..eadd284 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/StoreInterpreter.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/StoreInterpreter.scala @@ -41,6 +41,14 @@ trait StoreInterpreter[F[_]] extends (StoreADT ~> F) { def getForumCount: F[Int] + def getChats: F[Observable[CybozuDBChat]] + + def getChat(id: AnyId): F[Option[CybozuChat]] + + def storeChat(chat: CybozuDBChat, writeType: WriteType): F[AnyId] + + def getChatCount: F[Int] + def storeComment(comment: CybozuDBComment, commentType: CommentType, writeType: WriteType): F[AnyId] def storeComments(comments: Seq[CybozuDBComment], commentType: CommentType, writeType: WriteType): F[Seq[AnyId]] @@ -86,6 +94,10 @@ trait StoreInterpreter[F[_]] extends (StoreADT ~> F) { case GetEventCount => getEventCount case GetEvents => getEvents case StoreEvent(event, writeType) => storeEvent(event, writeType) + case GetChats => getChats + case GetChat(id) => getChat(id) + case GetChatCount => getChatCount + case StoreChat(chat, writeType) => storeChat(chat, writeType) case StoreComment(comment, commentType, writeType) => storeComment(comment, commentType, writeType) case StoreComments(comments, commentType, writeType) => storeComments(comments, commentType, writeType) case StoreTodoAssignees(todoId, assigneeIds) => storeTodoAssignees(todoId, assigneeIds) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/SQLiteInterpreter.scala b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/SQLiteInterpreter.scala index 0d50af3..78371d7 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/SQLiteInterpreter.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/SQLiteInterpreter.scala @@ -88,11 +88,30 @@ class SQLiteInterpreter(dbPath: Path)(implicit exc: Scheduler) extends StoreInte db.run(forumTableOps.count) } + override def getChats: Task[Observable[CybozuDBChat]] = Task.eval { + Observable.fromReactivePublisher( + db.stream(chatTableOps.stream) + ) + } + + override def getChat(id: AnyId): Task[Option[CybozuChat]] = Task.deferFuture { + db.run(chatTableOps.getChat(id)) + } + + override def storeChat(chat: CybozuDBChat, writeType: WriteType): Task[AnyId] = Task.deferFuture { + db.run(chatTableOps.write(chat, writeType)) + } + + override def getChatCount: Task[AnyId] = Task.deferFuture { + db.run(chatTableOps.count) + } + override def storeComment(comment: CybozuDBComment, commentType: CommentType, writeType: WriteType): Task[AnyId] = Task.deferFuture { commentType match { case TodoComment => db.run(todoCommentTableOps.write(comment, writeType)) case EventComment => db.run(eventCommentTableOps.write(comment, writeType)) case ForumComment => db.run(forumCommentTableOps.write(comment, writeType)) + case ChatComment => db.run(chatCommentTableOps.write(comment, writeType)) } } @@ -101,6 +120,7 @@ class SQLiteInterpreter(dbPath: Path)(implicit exc: Scheduler) extends StoreInte case TodoComment => db.run(todoCommentTableOps.write(comments, writeType)) case EventComment => db.run(eventCommentTableOps.write(comments, writeType)) case ForumComment => db.run(forumCommentTableOps.write(comments, writeType)) + case ChatComment => db.run(chatCommentTableOps.write(comments, writeType)) } } diff --git a/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/ChatTableOps.scala b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/ChatTableOps.scala index 6ecd66e..bb4dcc6 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/ChatTableOps.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/persistence/interpreters/sqlite/ops/ChatTableOps.scala @@ -15,6 +15,11 @@ private[sqlite] case class ChatTableOps()(implicit exec: ExecutionContext) exten private val commentTableQuery = TableQuery[ChatCommentTable] private val cybozuUserTableQuery = TableQuery[CybozuUserTable] + lazy val count: DBIORead[Int] = + tableQuery + .length + .result + def getChat(id: AnyId): DBIORead[Option[CybozuChat]] = for { optChat <- tableQuery From 85fcdfee8d27abb9fa7940b0469e00226460dcf1 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Thu, 23 Aug 2018 13:19:36 +0900 Subject: [PATCH 14/30] =?UTF-8?q?Fix:=20text=20files=20can=E2=80=99t=20loa?= =?UTF-8?q?d?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/scala/com/nulabinc/backlog/c2b/App.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/App.scala b/src/main/scala/com/nulabinc/backlog/c2b/App.scala index e8d13a5..d991220 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/App.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/App.scala @@ -103,7 +103,10 @@ object App extends Logger { val backlogApi = AllApi.accessKey(s"${config.backlogUrl}/api/v2/", config.backlogKey) - val csvFiles = Config.DATA_PATHS.toFile.listFiles().filter(_.getName.endsWith(".csv")) + val exportFiles = Config.DATA_PATHS + .toFile + .listFiles() + .filter(file => file.getName.endsWith(".csv") || file.getName.endsWith(".txt")) for { // Initialize @@ -116,7 +119,7 @@ object App extends Logger { _ <- AppDSL.fromStorage(StorageDSL.deleteFile(Config.DB_PATH)) _ <- AppDSL.fromStore(StoreDSL.createDatabase) // Read CSV and to store - _ <- CybozuStore.copyToStore(csvFiles) + _ <- CybozuStore.copyToStore(exportFiles) // Collect Backlog data to store _ <- BacklogService.storePriorities(backlogApi.priorityApi) _ <- BacklogService.storeStatuses(backlogApi.statusApi) From 77865a06a6c2f034165e6cee5de19c019fbf24e8 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Thu, 23 Aug 2018 15:14:40 +0900 Subject: [PATCH 15/30] Fix: test --- src/test/scala/com/nulabinc/backlog/c2b/AppSpec.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/test/scala/com/nulabinc/backlog/c2b/AppSpec.scala b/src/test/scala/com/nulabinc/backlog/c2b/AppSpec.scala index 89238b2..acef438 100644 --- a/src/test/scala/com/nulabinc/backlog/c2b/AppSpec.scala +++ b/src/test/scala/com/nulabinc/backlog/c2b/AppSpec.scala @@ -124,6 +124,14 @@ class AppSpec extends FlatSpec with Matchers { override def getCybozuStatuses(): Task[Observable[CybozuStatus]] = ??? override def writeDBStream[A](stream: Observable[StoreProgram[A]]): Task[Unit] = ??? + + override def getChats: Task[Observable[CybozuDBChat]] = ??? + + override def getChat(id: AnyId): Task[Option[CybozuChat]] = ??? + + override def storeChat(chat: CybozuDBChat, writeType: WriteType): Task[AnyId] = ??? + + override def getChatCount: Task[AnyId] = ??? } class TestConsoleInterpreter extends ConsoleInterpreter { From 0b579163c2b8daf574cc1abc3a88b8d719ce5854 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Thu, 23 Aug 2018 17:10:15 +0900 Subject: [PATCH 16/30] Add: topic parser --- .../backlog/c2b/parsers/TextFileParser.scala | 43 +++++++++++++++++++ .../c2b/parsers/TextFileParserSpec.scala | 11 +++++ 2 files changed, 54 insertions(+) create mode 100644 src/main/scala/com/nulabinc/backlog/c2b/parsers/TextFileParser.scala create mode 100644 src/test/scala/com/nulabinc/backlog/c2b/parsers/TextFileParserSpec.scala diff --git a/src/main/scala/com/nulabinc/backlog/c2b/parsers/TextFileParser.scala b/src/main/scala/com/nulabinc/backlog/c2b/parsers/TextFileParser.scala new file mode 100644 index 0000000..116d145 --- /dev/null +++ b/src/main/scala/com/nulabinc/backlog/c2b/parsers/TextFileParser.scala @@ -0,0 +1,43 @@ +package com.nulabinc.backlog.c2b.parsers + +import com.nulabinc.backlog.c2b.datas.{CybozuTextPost, CybozuTextTopic} + +object TextFileParser { + + private val postPattern = """(?ms)\r\n(\d+)?: (.+? .+?) (.+?)\r\n\r\n(.*?)\r\n\r\n--------------------------------------------------------------------------------""".r + private val MIN_TOPIC_LINES = 4 + private val TITLE_LINE_INDEX = 1 + private val DESCRIPRION_START_INDEX = 3 + + def topic(topicText: String): Either[ParseError[CybozuTextTopic], CybozuTextTopic] = { + val lines = topicText.split("\n") + + if (lines.length < MIN_TOPIC_LINES) + Left(CannotParseFromString(classOf[CybozuTextTopic], "Invalid topic rows: min", topicText)) + else { + val result = for { + title <- title(lines(TITLE_LINE_INDEX)) + description <- description(lines.slice(DESCRIPRION_START_INDEX, lines.length)) + } yield CybozuTextTopic(title, description) + result match { + case Right(value) => Right(value) + case Left(error) => Left(CannotParseFromString(classOf[CybozuTextTopic], error.getMessage, lines.mkString("\n"))) + } + } + } + + def post(postStr: String): Either[ParseError[CybozuTextPost], CybozuTextPost] = ??? + + private[parsers] def title(line: String): Either[Throwable, String] = { + val regex = """.+?: (.+?)""".r + + line match { + case regex(title) => Right(title) + case _ => Left(new RuntimeException("Cannot find topic title")) + } + } + + private[parsers] def description(lines: Seq[String]): Either[Throwable, String] = + Right(lines.mkString("\n")) + +} diff --git a/src/test/scala/com/nulabinc/backlog/c2b/parsers/TextFileParserSpec.scala b/src/test/scala/com/nulabinc/backlog/c2b/parsers/TextFileParserSpec.scala new file mode 100644 index 0000000..65f2e26 --- /dev/null +++ b/src/test/scala/com/nulabinc/backlog/c2b/parsers/TextFileParserSpec.scala @@ -0,0 +1,11 @@ +package com.nulabinc.backlog.c2b.parsers + +import org.scalatest.{FlatSpec, Matchers} + +class TextFileParserSpec extends FlatSpec with Matchers { + + "TextFileParser.title" should "extract title from line" in { + TextFileParser.title("Title: aaa") shouldEqual Right("aaa") + TextFileParser.title("タイトル: テスト") shouldEqual Right("テスト") + } +} From 729c67afb538c108d4838585a4ed3a2094e6e9a4 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Thu, 23 Aug 2018 17:10:51 +0900 Subject: [PATCH 17/30] Clean: add error message info --- .../scala/com/nulabinc/backlog/c2b/parsers/ParseError.scala | 2 +- .../nulabinc/backlog/c2b/parsers/ZonedDateTimeParser.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/parsers/ParseError.scala b/src/main/scala/com/nulabinc/backlog/c2b/parsers/ParseError.scala index 2b78530..7503082 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/parsers/ParseError.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/parsers/ParseError.scala @@ -5,7 +5,7 @@ import org.apache.commons.csv.CSVRecord sealed trait ParseError[A] case class CannotParseCSV[A](klass: Class[A], reason: String, record: CSVRecord) extends ParseError[A] -case class CannotParseFromString[A](klass: Class[A], value: String) extends ParseError[A] { +case class CannotParseFromString[A](klass: Class[A], reason: String, value: String) extends ParseError[A] { override def toString: String = s"$klass $value" } case class CannotParseComment(reason: String, data: String) extends ParseError[CybozuCSVComment] diff --git a/src/main/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParser.scala b/src/main/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParser.scala index 15e39a6..bede7ea 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParser.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParser.scala @@ -59,8 +59,8 @@ object ZonedDateTimeParser { Try(otherDateFormat.parse(value)) match { case Success(date) => Right(ZonedDateTime.ofInstant(date.toInstant, ZoneId.systemDefault())) - case Failure(_) => - Left(CannotParseFromString(classOf[DateTime], value)) + case Failure(error) => + Left(CannotParseFromString(classOf[DateTime], s"Cannot parse zoned date time: ${error.getMessage}", value)) } } } From 29241caff707097d8335ab95707ed1dda7329034 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Fri, 24 Aug 2018 12:39:03 +0900 Subject: [PATCH 18/30] Update: topic to store --- .../backlog/c2b/datas/CybozuDBModels.scala | 11 +++++- .../backlog/c2b/datas/CybozuModels.scala | 15 +++++++- .../backlog/c2b/parsers/ParseError.scala | 3 +- .../backlog/c2b/parsers/TextFileParser.scala | 31 +++++++++++----- .../c2b/parsers/ZonedDateTimeParser.scala | 18 ++++++++-- .../c2b/readers/CybozuTextReader.scala | 2 +- .../backlog/c2b/services/CybozuStore.scala | 36 +++++++++++++++++-- .../backlog/c2b/syntax/EitherOps.scala | 10 ++++++ 8 files changed, 110 insertions(+), 16 deletions(-) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuDBModels.scala b/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuDBModels.scala index ce964ab..9c95b53 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuDBModels.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuDBModels.scala @@ -47,7 +47,7 @@ object CybozuDBComment { val tupled = (this.apply _).tupled def from(parentIssueId: AnyId, comment: CybozuCSVComment, creatorId: AnyId): CybozuDBComment = - new CybozuDBComment( + CybozuDBComment( id = 0, parentId = parentIssueId, creator = creatorId, @@ -55,6 +55,15 @@ object CybozuDBComment { content = comment.content ) + def from(parentIssueId: AnyId, post: CybozuTextPost, postUserId: AnyId): CybozuDBComment = + CybozuDBComment( + id = 0, + parentId = parentIssueId, + creator = postUserId, + createdAt = post.postedAt, + content = post.content + ) + def to(comment: CybozuDBComment, creator: CybozuUser): CybozuComment = CybozuComment( id = comment.id, diff --git a/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuModels.scala b/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuModels.scala index 011deab..d52c23d 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuModels.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuModels.scala @@ -10,7 +10,9 @@ case class CybozuUser( object CybozuUser { val tupled = (this.apply _).tupled def from(user: CybozuCSVUser): CybozuUser = - new CybozuUser(0, user.value) + CybozuUser(0, user.value) + def fromCybozuTextUser(user: CybozuTextUser): CybozuUser = + CybozuUser(0, user.value) } case class CybozuTodo( @@ -65,6 +67,17 @@ case class CybozuComment( content: String ) +object CybozuComment { + def from(parentId: AnyId, post: CybozuTextPost, postUserId: AnyId): CybozuComment = + CybozuComment( + id = 0, + parentId = parentId, + creator = CybozuUser(postUserId, post.postUser.value), + createdAt = post.postedAt, + content = post.content + ) +} + case class CybozuIssueType(value: String) extends AnyVal case class CybozuStatus(value: String) extends AnyVal case class CybozuPriority(value: String) extends AnyVal diff --git a/src/main/scala/com/nulabinc/backlog/c2b/parsers/ParseError.scala b/src/main/scala/com/nulabinc/backlog/c2b/parsers/ParseError.scala index 7503082..68a7e9b 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/parsers/ParseError.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/parsers/ParseError.scala @@ -1,6 +1,6 @@ package com.nulabinc.backlog.c2b.parsers -import com.nulabinc.backlog.c2b.datas.CybozuCSVComment +import com.nulabinc.backlog.c2b.datas.{CybozuCSVComment, CybozuTextPost} import org.apache.commons.csv.CSVRecord sealed trait ParseError[A] @@ -9,3 +9,4 @@ case class CannotParseFromString[A](klass: Class[A], reason: String, value: Stri override def toString: String = s"$klass $value" } case class CannotParseComment(reason: String, data: String) extends ParseError[CybozuCSVComment] +case class CannotParsePost(reason: String, data: String) extends ParseError[CybozuTextPost] diff --git a/src/main/scala/com/nulabinc/backlog/c2b/parsers/TextFileParser.scala b/src/main/scala/com/nulabinc/backlog/c2b/parsers/TextFileParser.scala index 116d145..acb12e3 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/parsers/TextFileParser.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/parsers/TextFileParser.scala @@ -1,10 +1,11 @@ package com.nulabinc.backlog.c2b.parsers -import com.nulabinc.backlog.c2b.datas.{CybozuTextPost, CybozuTextTopic} +import com.nulabinc.backlog.c2b.datas.{CybozuTextPost, CybozuTextTopic, CybozuTextUser} object TextFileParser { - private val postPattern = """(?ms)\r\n(\d+)?: (.+? .+?) (.+?)\r\n\r\n(.*?)\r\n\r\n--------------------------------------------------------------------------------""".r + private val titlePattern = """.+?: (.+?)""".r + private val postPattern = """(?ms)\n(\d+)?: (.+? .+?)\n(.+?)\n\n(.*?)\n""".r private val MIN_TOPIC_LINES = 4 private val TITLE_LINE_INDEX = 1 private val DESCRIPRION_START_INDEX = 3 @@ -26,16 +27,30 @@ object TextFileParser { } } - def post(postStr: String): Either[ParseError[CybozuTextPost], CybozuTextPost] = ??? - - private[parsers] def title(line: String): Either[Throwable, String] = { - val regex = """.+?: (.+?)""".r + def post(postStr: String): Either[ParseError[CybozuTextPost], CybozuTextPost] = { + postStr match { + case postPattern(_, userString, postedAtString, message) => + val result = for { + postedAt <- ZonedDateTimeParser.toZonedDateTime(postedAtString) + } yield + CybozuTextPost( + content = message, + postUser = CybozuTextUser(userString), + postedAt = postedAt + ) + result match { + case Right(value) => Right(value) + case Left(error) => Left(CannotParsePost("Cannot parse post.", error.toString)) + } + case other => Left(CannotParsePost("Invalid post content", other)) + } + } + private[parsers] def title(line: String): Either[Throwable, String] = line match { - case regex(title) => Right(title) + case titlePattern(title) => Right(title) case _ => Left(new RuntimeException("Cannot find topic title")) } - } private[parsers] def description(lines: Seq[String]): Either[Throwable, String] = Right(lines.mkString("\n")) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParser.scala b/src/main/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParser.scala index bede7ea..2637d93 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParser.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/parsers/ZonedDateTimeParser.scala @@ -14,7 +14,8 @@ object ZonedDateTimeParser { def toZonedDateTime(value: String): Either[ParseError[DateTime], DateTime] = { val pattern1 = """(\d+?)/(\d+?)/(\d+?) .*?(\d+?):(\d+?):(\d+)""".r val pattern2 = """(\d+?)/(\d+?)/(\d+?) .*?(\d+?):(\d+?)""".r - val pattern3 = """(\d+?)/(\d+?)/(\d+?)""".r + val pattern3 = """(\d+?)/(\d+?)/(\d+?).+?(\d+?):(\d+?)""".r + val pattern4 = """(\d+?)/(\d+?)/(\d+?)""".r value match { case pattern1(year, month, day, hour, minutes, seconds) => Right( @@ -42,7 +43,20 @@ object ZonedDateTimeParser { ZoneId.systemDefault() ) ) - case pattern3(year, month, day) => + case pattern3(year, month, day, hour, minutes) => + Right( + ZonedDateTime.of( + year.toInt, + month.toInt, + day.toInt, + hour.toInt, + minutes.toInt, + 0, + 0, + ZoneId.systemDefault() + ) + ) + case pattern4(year, month, day) => Right( ZonedDateTime.of( year.toInt, diff --git a/src/main/scala/com/nulabinc/backlog/c2b/readers/CybozuTextReader.scala b/src/main/scala/com/nulabinc/backlog/c2b/readers/CybozuTextReader.scala index 654ea70..8f4cce9 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/readers/CybozuTextReader.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/readers/CybozuTextReader.scala @@ -33,7 +33,7 @@ object CybozuTopicTextReader { else Last(acc.topicText, acc.comments :+ acc.last, "") } else { - Last(acc.topicText, acc.comments, acc.last + line) + Last(acc.topicText, acc.comments, acc.last + "\n" + line) } }.map { result => TopicReadResult(result.topicText, result.comments) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/services/CybozuStore.scala b/src/main/scala/com/nulabinc/backlog/c2b/services/CybozuStore.scala index 692a86b..95a9d79 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/services/CybozuStore.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/services/CybozuStore.scala @@ -5,12 +5,16 @@ import java.io.File import com.nulabinc.backlog.c2b.core.Logger import com.nulabinc.backlog.c2b.datas._ import com.nulabinc.backlog.c2b.datas.Types.AnyId +import com.nulabinc.backlog.c2b.exceptions.CybozuLiveImporterException import com.nulabinc.backlog.c2b.interpreters.{AppDSL, ConsoleDSL} import com.nulabinc.backlog.c2b.interpreters.AppDSL.AppProgram +import com.nulabinc.backlog.c2b.parsers.TextFileParser import com.nulabinc.backlog.c2b.persistence.dsl._ import com.nulabinc.backlog.c2b.persistence.dsl.StoreDSL.StoreProgram -import com.nulabinc.backlog.c2b.readers.CybozuCSVReader +import com.nulabinc.backlog.c2b.readers.{CybozuCSVReader, CybozuTopicTextReader} import com.osinka.i18n.Messages +import monix.eval.Task +import monix.reactive.Observable object CybozuStore extends Logger { @@ -123,10 +127,38 @@ object CybozuStore extends Logger { ) } yield () - def chat(files: Array[File]): AppProgram[Unit] = + def chat(files: Array[File]): AppProgram[Unit] = { + import com.nulabinc.backlog.c2b.syntax.EitherOps._ + for { _ <- AppDSL.fromConsole(ConsoleDSL.print(Messages("message.init.collect", Messages("name.chat")))) + _ <- AppDSL.fromStore( + StoreDSL.writeDBStream( + CybozuTopicTextReader.read(files).map { result => + val topic = TextFileParser.topic(result.topicText).orExit + val postStream = result.comments.map(TextFileParser.post).map { + case Right(value) => value + case Left(error) => throw CybozuLiveImporterException(error.toString) + } + + for { + chatId <- StoreDSL.storeChat(CybozuDBChat.from(topic)) + _ <- StoreDSL.writeDBStream { + val stream = postStream.map { post => + val creator = CybozuUser.fromCybozuTextUser(post.postUser) + for { + postUserId <- insertOrUpdateUser(creator) + _ <- StoreDSL.storeComment(CybozuDBComment.from(chatId, post, postUserId), ChatComment) + } yield () + } + Observable.fromIterator(stream.toIterator) + } + } yield () + } + ) + ) } yield () + } private def sequential[A](prgs: Seq[StoreProgram[A]]): StoreProgram[Seq[A]] = prgs.foldLeft(StoreDSL.pure(Seq.empty[A])) { diff --git a/src/main/scala/com/nulabinc/backlog/c2b/syntax/EitherOps.scala b/src/main/scala/com/nulabinc/backlog/c2b/syntax/EitherOps.scala index 61f9cd0..66d37eb 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/syntax/EitherOps.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/syntax/EitherOps.scala @@ -1,5 +1,7 @@ package com.nulabinc.backlog.c2b.syntax +import com.nulabinc.backlog.c2b.exceptions.CybozuLiveImporterException + object EitherOps { implicit class SeqEitherOps[E, B](results: Seq[Either[E, B]]) { def sequence: Either[E, Seq[B]] = @@ -8,4 +10,12 @@ object EitherOps { case (acc, Right(item)) => acc.map(_ :+ item) } } + + implicit class ErrorEitherOps[E, B](result: Either[E, B]) { + def orExit: B = + result match { + case Right(value) => value + case Left(error) => throw CybozuLiveImporterException(error.toString) + } + } } From 2bb96ab124e3763bbb572c9e484b9820944df037 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Fri, 24 Aug 2018 15:17:40 +0900 Subject: [PATCH 19/30] Add: define issue type --- src/main/resources/messages.txt | 1 + src/main/resources/messages_ja.txt | 1 + src/main/scala/com/nulabinc/backlog/c2b/datas/IssueType.scala | 1 + .../scala/com/nulabinc/backlog/c2b/services/CybozuStore.scala | 1 - 4 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/resources/messages.txt b/src/main/resources/messages.txt index 747c1f0..fe6d683 100644 --- a/src/main/resources/messages.txt +++ b/src/main/resources/messages.txt @@ -37,6 +37,7 @@ validation.backlog_project_already_exist=Project "{0}" already exists. Do you wa issue.type.todo=To-Do issue.type.event=Event issue.type.forum=Forum +issue.type.chat=Chat # Mapping file mapping.output_file=The mapping file about {0} is created. Check the mapping file and fix it if necessary. diff --git a/src/main/resources/messages_ja.txt b/src/main/resources/messages_ja.txt index 1682117..682578b 100644 --- a/src/main/resources/messages_ja.txt +++ b/src/main/resources/messages_ja.txt @@ -37,6 +37,7 @@ validation.backlog_project_already_exist=プロジェクト[{0}]はBacklog内に issue.type.todo=ToDo issue.type.event=イベント issue.type.forum=掲示板 +issue.type.chat=チャット # Mapping file mapping.output_file={0}のマッピングファイルを作成しました。マッピングファイルを確認し、必要に応じて修正してください。 diff --git a/src/main/scala/com/nulabinc/backlog/c2b/datas/IssueType.scala b/src/main/scala/com/nulabinc/backlog/c2b/datas/IssueType.scala index b3512c0..c1cb3f1 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/datas/IssueType.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/datas/IssueType.scala @@ -6,4 +6,5 @@ object IssueType { case object ToDo extends IssueType case object Event extends IssueType case object Forum extends IssueType + case object Chat extends IssueType } diff --git a/src/main/scala/com/nulabinc/backlog/c2b/services/CybozuStore.scala b/src/main/scala/com/nulabinc/backlog/c2b/services/CybozuStore.scala index 95a9d79..32d88a2 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/services/CybozuStore.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/services/CybozuStore.scala @@ -13,7 +13,6 @@ import com.nulabinc.backlog.c2b.persistence.dsl._ import com.nulabinc.backlog.c2b.persistence.dsl.StoreDSL.StoreProgram import com.nulabinc.backlog.c2b.readers.{CybozuCSVReader, CybozuTopicTextReader} import com.osinka.i18n.Messages -import monix.eval.Task import monix.reactive.Observable object CybozuStore extends Logger { From 13ebdfcedb3b864a55b662b41581445cbeaf5be1 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Fri, 24 Aug 2018 15:18:50 +0900 Subject: [PATCH 20/30] Update: import chat as Backlog issue --- .../c2b/converters/IssueConverter.scala | 10 ++++ .../backlog/c2b/datas/CybozuModels.scala | 5 +- .../backlog/c2b/services/BacklogExport.scala | 48 ++++++++++++++++++- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/converters/IssueConverter.scala b/src/main/scala/com/nulabinc/backlog/c2b/converters/IssueConverter.scala index 00fb3c6..f027d29 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/converters/IssueConverter.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/converters/IssueConverter.scala @@ -93,6 +93,16 @@ class IssueConverter()(implicit ctx: MappingContext) extends Logger { } } + def from(from: CybozuChat, issueType: CybozuIssueType): Either[ConvertError, BacklogIssue] = + Right( + defaultBacklogIssue.copy( + id = from.id, + summary = createBacklogIssueSummary(from.title), + description = from.description, + optIssueTypeName = Some(issueType.value) + ) + ) + private def createBacklogIssueSummary(summary: String): BacklogIssueSummary = BacklogIssueSummary(value = summary, original = summary) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuModels.scala b/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuModels.scala index d52c23d..4f98e23 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuModels.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/datas/CybozuModels.scala @@ -1,5 +1,7 @@ package com.nulabinc.backlog.c2b.datas +import java.time.{ZoneId, ZonedDateTime} + import com.nulabinc.backlog.c2b.datas.Types.{AnyId, DateTime} case class CybozuUser( @@ -56,7 +58,8 @@ case class CybozuChat( id: AnyId, title: String, description: String, - comments: Seq[CybozuComment] + comments: Seq[CybozuComment], + createdAt: DateTime = ZonedDateTime.now(ZoneId.systemDefault) ) case class CybozuComment( diff --git a/src/main/scala/com/nulabinc/backlog/c2b/services/BacklogExport.scala b/src/main/scala/com/nulabinc/backlog/c2b/services/BacklogExport.scala index afef1d4..b15a110 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/services/BacklogExport.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/services/BacklogExport.scala @@ -27,7 +27,8 @@ object BacklogExport extends Logger { val issueTypes: Map[IssueType, CybozuIssueType] = Map( IssueType.ToDo -> CybozuIssueType(Messages("issue.type.todo")), IssueType.Event -> CybozuIssueType(Messages("issue.type.event")), - IssueType.Forum -> CybozuIssueType(Messages("issue.type.forum")) + IssueType.Forum -> CybozuIssueType(Messages("issue.type.forum")), + IssueType.Chat -> CybozuIssueType(Messages("issue.type.chat")) ) val openStatusName: String = Messages("name.status.open") @@ -43,7 +44,8 @@ object BacklogExport extends Logger { _ <- customFields(config) todoCount <- todos(config, issueTypes(IssueType.ToDo), openStatusName, 0) eventCount <- events(config, issueTypes(IssueType.Event), todoCount) - _ <- forums(config, issueTypes(IssueType.Forum), eventCount) + forumCount <- forums(config, issueTypes(IssueType.Forum), eventCount) + _ <- chats(config, issueTypes(IssueType.Chat), forumCount) _ <- finish } yield () @@ -174,6 +176,22 @@ object BacklogExport extends Logger { } yield total } + def chats(config: Config, issueType: CybozuIssueType, startIndex: Int)(implicit mappingContext: MappingContext): AppProgram[Int] = { + val issueConverter = new IssueConverter() + val commentConverter = new CommentConverter() + + for { + chats <- AppDSL.fromStore(StoreDSL.getChats) + total <- AppDSL.fromStore(StoreDSL.getChatCount) + _ <- AppDSL.consumeStream { + chats.zipWithIndex.map { + case (chat, index) => + exportChat(config.backlogPaths, chat.id, issueType, issueConverter, commentConverter, startIndex + index, startIndex + total) + } + } + } yield total + } + private def exportTodo(paths: BacklogPaths, todoId: AnyId, issueType: CybozuIssueType, @@ -271,6 +289,32 @@ object BacklogExport extends Logger { } yield () + private def exportChat(paths: BacklogPaths, + chatId: AnyId, + issueType: CybozuIssueType, + issueConverter: IssueConverter, + commentConverter: CommentConverter, + index: Long, + total: Long): AppProgram[Unit] = + for { + optChat <- AppDSL.fromStore(StoreDSL.getChat(chatId)) + _ <- optChat.map { chat => + val newId = index.toInt + 1 + val comments = chat.comments.map(c => c.copy(parentId = newId)) + chat.copy(id = newId, comments = comments, createdAt = comments.headOption.map(_.createdAt).getOrElse(chat.createdAt)) + }.map { chat => + issueConverter.from(chat.copy(id = index.toInt + 1), issueType) match { + case Right(backlogIssue) => + for { + _ <- exportIssue(paths, backlogIssue, chat.createdAt, index, total) + _ <- exportComments(paths, chat.comments, commentConverter) + } yield () + case Left(error) => + throw CybozuLiveImporterException("Chat convert error. " + error.toString) + } + }.getOrElse(throw CybozuLiveImporterException("Chat not found")) + } yield () + private def exportIssue(paths: BacklogPaths, backlogIssue: BacklogIssue, createdAt: DateTime, From 3d52bde2e47eb1e231d53f4dd89be53cdaca7170 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Fri, 24 Aug 2018 20:12:45 +0900 Subject: [PATCH 21/30] FIx: supports multi line --- .../backlog/c2b/parsers/TextFileParser.scala | 53 ++++++++++++------- .../c2b/parsers/TextFileParserSpec.scala | 5 ++ 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/parsers/TextFileParser.scala b/src/main/scala/com/nulabinc/backlog/c2b/parsers/TextFileParser.scala index acb12e3..edfd836 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/parsers/TextFileParser.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/parsers/TextFileParser.scala @@ -5,11 +5,17 @@ import com.nulabinc.backlog.c2b.datas.{CybozuTextPost, CybozuTextTopic, CybozuTe object TextFileParser { private val titlePattern = """.+?: (.+?)""".r - private val postPattern = """(?ms)\n(\d+)?: (.+? .+?)\n(.+?)\n\n(.*?)\n""".r private val MIN_TOPIC_LINES = 4 private val TITLE_LINE_INDEX = 1 private val DESCRIPRION_START_INDEX = 3 + // post + private val MIN_POST_LINES = 5 + private val userPattern = """\d+?: (.+?)""".r + private val USER_LINE_INDEX = 1 + private val POSTED_AT_LINE_INDEX = 2 + private val CONTENT_START_INDEX = 4 + def topic(topicText: String): Either[ParseError[CybozuTextTopic], CybozuTextTopic] = { val lines = topicText.split("\n") @@ -18,7 +24,7 @@ object TextFileParser { else { val result = for { title <- title(lines(TITLE_LINE_INDEX)) - description <- description(lines.slice(DESCRIPRION_START_INDEX, lines.length)) + description = arrayToString(lines.slice(DESCRIPRION_START_INDEX, lines.length)) } yield CybozuTextTopic(title, description) result match { case Right(value) => Right(value) @@ -28,21 +34,24 @@ object TextFileParser { } def post(postStr: String): Either[ParseError[CybozuTextPost], CybozuTextPost] = { - postStr match { - case postPattern(_, userString, postedAtString, message) => - val result = for { - postedAt <- ZonedDateTimeParser.toZonedDateTime(postedAtString) - } yield - CybozuTextPost( - content = message, - postUser = CybozuTextUser(userString), - postedAt = postedAt - ) - result match { - case Right(value) => Right(value) - case Left(error) => Left(CannotParsePost("Cannot parse post.", error.toString)) - } - case other => Left(CannotParsePost("Invalid post content", other)) + val lines = postStr.split("\n") + + if (lines.length < MIN_POST_LINES) + Left(CannotParsePost("Invalid post rows: min", postStr)) + else { + val dateString = lines(POSTED_AT_LINE_INDEX) + val postedAtResult = ZonedDateTimeParser.toZonedDateTime(dateString) match { + case Right(value) => Right(value) + case Left(error) => Left(CannotParseFromString(classOf[CybozuTextPost], error.toString, dateString)) + } + for { + postedAt <- postedAtResult + user <- user(lines(USER_LINE_INDEX)) + } yield CybozuTextPost( + content = arrayToString(lines.slice(CONTENT_START_INDEX, lines.length)), + postUser = CybozuTextUser(user), + postedAt = postedAt + ) } } @@ -52,7 +61,13 @@ object TextFileParser { case _ => Left(new RuntimeException("Cannot find topic title")) } - private[parsers] def description(lines: Seq[String]): Either[Throwable, String] = - Right(lines.mkString("\n")) + private[parsers] def arrayToString(lines: Seq[String]): String = + lines.mkString("\n") + + private[parsers] def user(line: String): Either[ParseError[CybozuTextPost], String] = + line match { + case userPattern(user) => Right(user) + case _ => Left(CannotParseFromString(classOf[CybozuTextPost], "Cannot find user string", line)) + } } diff --git a/src/test/scala/com/nulabinc/backlog/c2b/parsers/TextFileParserSpec.scala b/src/test/scala/com/nulabinc/backlog/c2b/parsers/TextFileParserSpec.scala index 65f2e26..df1a8f6 100644 --- a/src/test/scala/com/nulabinc/backlog/c2b/parsers/TextFileParserSpec.scala +++ b/src/test/scala/com/nulabinc/backlog/c2b/parsers/TextFileParserSpec.scala @@ -8,4 +8,9 @@ class TextFileParserSpec extends FlatSpec with Matchers { TextFileParser.title("Title: aaa") shouldEqual Right("aaa") TextFileParser.title("タイトル: テスト") shouldEqual Right("テスト") } + + "TextFileParser.user" should "extract username from line" in { + TextFileParser.user("23: Shoma Nishitaten") shouldEqual Right("Shoma Nishitaten") + TextFileParser.user("23: Shoma :Nishitaten") shouldEqual Right("Shoma :Nishitaten") + } } From 23369949be87a5f58c5bcf8c4a90dd644c79b6d7 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Sat, 25 Aug 2018 17:36:51 +0900 Subject: [PATCH 22/30] Add: limitation about Chat migration --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index e18bd24..efb8173 100644 --- a/README.md +++ b/README.md @@ -101,11 +101,16 @@ Backlogの **管理者権限** が必要になります。 ### プロジェクトについて * テキスト整形のルール: **markdown** +### チャットデータの移行について +* 状態は全て **Open** となります。 + ### CybozuLive側の制限について - 掲示板/ToDoリスト - コメント数:最新から10,000件 - イベント - コメント数:最新から10,000件 +- チャット + - コメント数:最新から10,000件 - 掲示板やイベントの添付ファイルは移行できません - ToDoのカテゴリは移行できません。 @@ -253,11 +258,16 @@ Sample commands: ### Backlog's user roles This program is for the users with the Space's **administrator** roles. +### About migrating chat data +* All states are **Open**. + ### About limitations in CybozuLive - Forum/ToDo-list - Comments:10,000 from the latest - Event - Comments:10,000 from the latest +- Chat + - Comments:10,000 from the latest - Can not migrate bulletin forum and event attachments - The ToDo category can not be migrated. From 0e46fe2c6179d790ae2aa561f6d7c408c40d9394 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Tue, 28 Aug 2018 12:27:46 +0900 Subject: [PATCH 23/30] Clean: description --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index efb8173..0b205b2 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ Backlogの **管理者権限** が必要になります。 * テキスト整形のルール: **markdown** ### チャットデータの移行について -* 状態は全て **Open** となります。 +* チャットとして移行された課題の状態は全て **Open** となります。 ### CybozuLive側の制限について - 掲示板/ToDoリスト @@ -259,7 +259,7 @@ Sample commands: This program is for the users with the Space's **administrator** roles. ### About migrating chat data -* All states are **Open**. +* All issues that are created when migrating chats will have state **Open**. ### About limitations in CybozuLive - Forum/ToDo-list From 59941ae91782b11ddc8d5fccdbf33bfafead78f8 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Tue, 28 Aug 2018 12:45:13 +0900 Subject: [PATCH 24/30] Fix: can not parse a delimiter that is similar to a delimiter line --- .../com/nulabinc/backlog/c2b/readers/CybozuTextReader.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/readers/CybozuTextReader.scala b/src/main/scala/com/nulabinc/backlog/c2b/readers/CybozuTextReader.scala index 8f4cce9..5118286 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/readers/CybozuTextReader.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/readers/CybozuTextReader.scala @@ -27,7 +27,7 @@ object CybozuTopicTextReader { .drop(1) .foldLeftF(Last("", Stream.empty[String], "")) { case (acc, line) => - if (line.startsWith(topicSeparator) || line.startsWith(commentSeparator)) { + if (line == topicSeparator || line == commentSeparator) { if (acc.topicText.isEmpty) Last(acc.last, acc.comments, "") else From 7106a5324691a8d5128986cf2a57c88ff09c8a5c Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Tue, 28 Aug 2018 13:29:37 +0900 Subject: [PATCH 25/30] Fix: wrong issue created date --- .../com/nulabinc/backlog/c2b/converters/IssueConverter.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/converters/IssueConverter.scala b/src/main/scala/com/nulabinc/backlog/c2b/converters/IssueConverter.scala index f027d29..1936a01 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/converters/IssueConverter.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/converters/IssueConverter.scala @@ -99,7 +99,10 @@ class IssueConverter()(implicit ctx: MappingContext) extends Logger { id = from.id, summary = createBacklogIssueSummary(from.title), description = from.description, - optIssueTypeName = Some(issueType.value) + optIssueTypeName = Some(issueType.value), + operation = defaultBacklogIssue.operation.copy( + optCreated = Some(DateUtil.toDateTimeString(from.createdAt)) + ) ) ) From 2ba871ab281287ed1dfd9a8612d9c3ce477856d6 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Tue, 28 Aug 2018 18:35:34 +0900 Subject: [PATCH 26/30] Fix: use first commented data --- .../c2b/converters/IssueConverter.scala | 14 ++++++++--- .../backlog/c2b/services/BacklogExport.scala | 25 +++++++++++-------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/converters/IssueConverter.scala b/src/main/scala/com/nulabinc/backlog/c2b/converters/IssueConverter.scala index 1936a01..7a615da 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/converters/IssueConverter.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/converters/IssueConverter.scala @@ -93,18 +93,24 @@ class IssueConverter()(implicit ctx: MappingContext) extends Logger { } } - def from(from: CybozuChat, issueType: CybozuIssueType): Either[ConvertError, BacklogIssue] = - Right( + def from(from: CybozuChat, firstPostUser: Option[CybozuUser], issueType: CybozuIssueType): Either[ConvertError, BacklogIssue] = { + val createdAt = DateUtil.toDateTimeString(from.createdAt) + for { + optCreator <- userConverter.to(firstPostUser) + } yield defaultBacklogIssue.copy( id = from.id, summary = createBacklogIssueSummary(from.title), description = from.description, optIssueTypeName = Some(issueType.value), operation = defaultBacklogIssue.operation.copy( - optCreated = Some(DateUtil.toDateTimeString(from.createdAt)) + optCreatedUser = optCreator, + optCreated = Some(createdAt), + optUpdatedUser = optCreator, + optUpdated = Some(createdAt) ) ) - ) + } private def createBacklogIssueSummary(summary: String): BacklogIssueSummary = BacklogIssueSummary(value = summary, original = summary) diff --git a/src/main/scala/com/nulabinc/backlog/c2b/services/BacklogExport.scala b/src/main/scala/com/nulabinc/backlog/c2b/services/BacklogExport.scala index b15a110..c4aba3c 100644 --- a/src/main/scala/com/nulabinc/backlog/c2b/services/BacklogExport.scala +++ b/src/main/scala/com/nulabinc/backlog/c2b/services/BacklogExport.scala @@ -301,17 +301,20 @@ object BacklogExport extends Logger { _ <- optChat.map { chat => val newId = index.toInt + 1 val comments = chat.comments.map(c => c.copy(parentId = newId)) - chat.copy(id = newId, comments = comments, createdAt = comments.headOption.map(_.createdAt).getOrElse(chat.createdAt)) - }.map { chat => - issueConverter.from(chat.copy(id = index.toInt + 1), issueType) match { - case Right(backlogIssue) => - for { - _ <- exportIssue(paths, backlogIssue, chat.createdAt, index, total) - _ <- exportComments(paths, chat.comments, commentConverter) - } yield () - case Left(error) => - throw CybozuLiveImporterException("Chat convert error. " + error.toString) - } + val optCreator = comments.headOption.map(_.creator) + val newChat = chat.copy(id = newId, comments = comments, createdAt = comments.headOption.map(_.createdAt).getOrElse(chat.createdAt)) + (newChat, optCreator) + }.map { + case (chat, optCreator) => + issueConverter.from(chat.copy(id = index.toInt + 1), optCreator, issueType) match { + case Right(backlogIssue) => + for { + _ <- exportIssue(paths, backlogIssue, chat.createdAt, index, total) + _ <- exportComments(paths, chat.comments, commentConverter) + } yield () + case Left(error) => + throw CybozuLiveImporterException("Chat convert error. " + error.toString) + } }.getOrElse(throw CybozuLiveImporterException("Chat not found")) } yield () From b5899214144761680563b1d640057946a9b316c4 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Tue, 28 Aug 2018 18:48:24 +0900 Subject: [PATCH 27/30] Add: --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 0b205b2..4d2c14a 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ Backlogの **管理者権限** が必要になります。 ### チャットデータの移行について * チャットとして移行された課題の状態は全て **Open** となります。 +* 課題の作成者と作成日時は、最初にチャットルームで投稿したユーザと投稿日時になります ### CybozuLive側の制限について - 掲示板/ToDoリスト @@ -260,6 +261,7 @@ This program is for the users with the Space's **administrator** roles. ### About migrating chat data * All issues that are created when migrating chats will have state **Open**. +* The creator of the issue and the creation date are the user who posted in the chat room and the posting date. ### About limitations in CybozuLive - Forum/ToDo-list From 7e1b60db44a0500b920691fb78d6ed6004e16124 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Tue, 28 Aug 2018 18:49:07 +0900 Subject: [PATCH 28/30] Fix: --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4d2c14a..32d54f7 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ Backlogの **管理者権限** が必要になります。 ### チャットデータの移行について * チャットとして移行された課題の状態は全て **Open** となります。 -* 課題の作成者と作成日時は、最初にチャットルームで投稿したユーザと投稿日時になります +* 課題の作成者と作成日時は、最初にチャットルームで投稿したユーザと投稿日時になります。 ### CybozuLive側の制限について - 掲示板/ToDoリスト From ad837f1823bde448eff634811c016037924d8c33 Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Wed, 29 Aug 2018 11:33:47 +0900 Subject: [PATCH 29/30] Fix: --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32d54f7..3071629 100644 --- a/README.md +++ b/README.md @@ -261,7 +261,7 @@ This program is for the users with the Space's **administrator** roles. ### About migrating chat data * All issues that are created when migrating chats will have state **Open**. -* The creator of the issue and the creation date are the user who posted in the chat room and the posting date. +* The user that first posted to the chat will be set as the issues Registered by and the first message post time will be set as the issues Created date. ### About limitations in CybozuLive - Forum/ToDo-list From d01c40dbd88b98d0c02b20f9b1a564ea00e4332f Mon Sep 17 00:00:00 2001 From: Shoma Nishitateno Date: Wed, 29 Aug 2018 11:40:01 +0900 Subject: [PATCH 30/30] Release: 1.1.0 --- build.sbt | 2 +- src/main/resources/application.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 10d5cba..2196b3f 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ name := "backlog-migration-cybozulive" lazy val commonSettings = Seq( - version := "1.0.1-SNAPSHOT", + version := "1.1.0", scalaVersion := "2.12.6" ) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 0f7673b..f03d18e 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -1,7 +1,7 @@ application { name = "Backlog Migration for CybozuLive" - version = "1.0.1-SNAPSHOT" + version = "1.1.0" title = ${application.name} ${application.version} (c) nulab.inc fileName = backlog-migration-cybozulive-${application.version}.jar language = default