iss #22: fetch repos with github client that jush work (without timeo…

…uts, cache, unittests ...)

app/hedgehog/Config.scala

@@ -15,7 +15,10 @@ import hedgehog.models.Sources
 class Config (appConfiguration: Configuration, asyncContext: ExecutionContext) {
   val apiBaseUrl = appConfiguration.getString("github.api_url")
                    .getOrElse("https://api.github.com").stripSuffix("/")
-  lazy val githubClientConfig = new hedgehog.clients.github.Config(apiBaseUrl)
+  lazy val githubClientConfig = new hedgehog.clients.github.Config(
+    apiBaseUrl,
+    accessToken = appConfiguration.getString("github.app_token")
+  )
   lazy val githubClient = new github.Client(githubClientConfig, asyncContext) //TODO: is there good place for that?
 
 

app/hedgehog/clients/github/Client.scala

@@ -1,23 +1,87 @@
 package hedgehog.clients.github
 
 import scala.concurrent.{Future, Promise}
-import hedgehog.models.{AccountSettings, Repo}
+import play.api.libs.ws.WS
+import play.api.libs.json.{JsSuccess, JsError}
 
+import hedgehog.models.{AccountSettings, Repo}
 
 /**
  * Copyright (c) Nikita Kovaliov, maizy.ru, 2014
  * See LICENSE.txt for details.
  */
 class Client(val config: Config, implicit val context: scala.concurrent.ExecutionContext) {
 
+  val PER_PAGE = 100
+
   def getRepos(accountSettings: AccountSettings): Future[Seq[Repo]] = {
-    val finish = Promise[Seq[Repo]]()
+    val resultPromise = Promise[Seq[Repo]]()
 
-    finish success List(new Repo("a", accountSettings.account),
-                        new Repo("b", accountSettings.account))
+    val wrapError: PartialFunction[Throwable, Unit] = {
+      //TODO: custom error subclasses
+      case err => resultPromise.failure(new FetchError(cause = err))
+    }
+    val getFirstPage = getPage(accountSettings, page = 1, perPage = PER_PAGE)
 
-    // finish failure new ConfigError("ou!")
+    getFirstPage onSuccess {
+      case PageRes(res, None, None) => resultPromise.success(res)
+      case PageRes(firstPageRes, _, Some(last)) =>
+        require(last > 1)
+        val otherPagesRes = Future.sequence(
+          for {
+            page <- 2 to last
+          } yield getPage(accountSettings, page, perPage = PER_PAGE)
+        )
+        otherPagesRes onSuccess {
+          case otherPages => resultPromise success (firstPageRes ++ otherPages.flatMap(_.repos))
+        }
+        otherPagesRes onFailure wrapError
+    }
+    getFirstPage onFailure wrapError
 
-    finish.future
+    resultPromise.future
   }
+
+  private def getPage(accountSettings: AccountSettings, page: Int, perPage: Int): Future[PageRes] = {
+    val account = accountSettings.account
+    val url = config.replaceBaseUrl(account.apiReposUrl)
+    val httpClient = (
+        WS.url(url).withQueryString(
+          "per_page" -> perPage.toString,
+          "page" -> page.toString,
+          "type" -> "open"
+        )
+        match {
+          case h if accountSettings.includePrivateRepos => h.withQueryString("type" ->"private", "type" -> "open")
+          case h => h
+        }
+      ) match {
+        case h if config.accessToken.isDefined => h.withHeaders("Authorization" -> s"token ${config.accessToken.get}")
+        case h => h
+      }
+
+    httpClient.get map {
+      case response =>
+        response.status match {
+          case 200 =>
+            GithubReades.repoSeqReads.reads(response.json) match {
+              case JsSuccess(res, _) =>
+                val rels = response.header("Link").map(parseLinkHeader)
+                PageRes(
+                  res,
+                  nextPage = rels.flatMap(_.get("next").flatMap(_.page)),
+                  lastPage = rels.flatMap(_.get("last")).flatMap(_.page))
+              case JsError(err) => throw new FetchError(s"Unable to parse repo json $err")
+            }
+          //TODO: custom exceptions subclasses
+          case code: Int if code >= 400 && code < 599 => throw new FetchError("No data for repos request")
+          case _ => throw new FetchError("Unknown error when fetching repos")
+        }
+    }
+  }
+}
+
+
+private case class PageRes(repos: Seq[Repo], nextPage: Option[Int], lastPage: Option[Int]) {
+  val currentPage: Option[Int] = nextPage map (_ - 1)
 }

app/hedgehog/clients/github/Config.scala

@@ -7,7 +7,7 @@ import java.util.regex.Pattern
  * See LICENSE.txt for details.
  */
 
-class Config(val apiBaseUrl: String) {
+class Config(val apiBaseUrl: String, val accessToken: Option[String]) {
 
   def replaceBaseUrl(url: String): String = {
     var processedUrl = url

app/hedgehog/clients/github/GithubReades.scala

@@ -0,0 +1,44 @@
+package hedgehog.clients.github
+
+import play.api.libs.functional.syntax._
+import play.api.libs.json.{Reads, __}
+
+import hedgehog.models.{GithubOrg, Account, GithubUser, Repo, ProgrammingLang}
+/**
+ * Copyright (c) Nikita Kovaliov, maizy.ru, 2014
+ * See LICENSE.txt for details.
+ */
+object GithubReades {
+
+  val accountReads: Reads[Account] = (
+    (__ \ "type").read[String] ~
+    (__ \ "login").read[String] ~
+    (__ \ "avatar_url").readNullable[String]
+  ) apply {
+    (accountType, login, avatarUrl) => accountType match {
+      case "User" => GithubUser(login, avatarUrl)
+      case "Organization" => GithubOrg(login, avatarUrl)
+      case _ => throw new FormatError(s"Unknown owner type "+ accountType)
+    }
+  }
+
+  val repoReads: Reads[Repo] = (
+    (__ \ "name").read[String] ~
+    (__ \ "owner").read[Account](accountReads) ~
+    (__ \ "description").readNullable[String] ~
+    (__ \ "private").read[Boolean] ~
+    (__ \ "language").readNullable[String]
+  ) apply {
+    (name, owner, description, isPrivate, lang) => {
+      Repo(
+        name,
+        owner,
+        description = description,
+        isPrivate = Some(isPrivate),
+        primaryLang = lang.map(l => ProgrammingLang(l.toLowerCase))
+      )
+    }
+  }
+
+  val repoSeqReads: Reads[Seq[Repo]] = Reads.seq[Repo](repoReads)
+}

app/hedgehog/clients/github/errors.scala

@@ -4,5 +4,6 @@ package hedgehog.clients.github
  * Copyright (c) Nikita Kovaliov, maizy.ru, 2014
  * See LICENSE.txt for details.
  */
-class Error(msg: String) extends Exception(msg)
-class FetchError(msg: String) extends Exception(msg)
+class Error(msg: String = null, cause: Throwable = null) extends Exception(msg, cause)
+class FetchError(msg: String = null, cause: Throwable = null) extends Error(msg, cause)
+class FormatError(msg: String = null, cause: Throwable = null) extends Error(msg, cause)

app/hedgehog/models/Account.scala

@@ -1,9 +1,8 @@
 package hedgehog.models
 
-import play.api.libs.json.{Json, JsValue, Writes}
+import play.api.libs.json._
 import hedgehog.UrlUtils.urlEncodePathSegment
 
-
 /**
  * Copyright (c) Nikita Kovaliov, maizy.ru, 2014
  * See LICENSE.txt for details.
@@ -30,6 +29,8 @@ trait Account {
 
   def getRepoUrl(repo: Repo): String =
     s"$webProfileUrl/${urlEncodePathSegment(repo.fullName)}"
+
+  def apiReposUrl: String
 }
 
 
@@ -54,6 +55,7 @@ case class GithubUser(
   extends Account {
 
   val accountType = AccountType.User
+  val apiReposUrl = s"$GITHUB_BASE_API_URL/users/${urlEncodePathSegment(name)}/repos"
 }
 
 
@@ -63,4 +65,5 @@ case class GithubOrg(
   extends Account {
 
   val accountType = AccountType.Org
+  val apiReposUrl = s"$GITHUB_BASE_API_URL/orgs/${urlEncodePathSegment(name)}/repos"
 }

app/hedgehog/models/Repo.scala

@@ -5,25 +5,23 @@ import play.api.libs.json._
  * Copyright (c) Nikita Kovaliov, maizy.ru, 2013-2014
  * See LICENSE.txt for details.
  */
-class Repo(
-    val name: String,
-    val owner: Account,
-    val description: Option[Boolean] = None,
-    val isPrivate: Option[Boolean] = None,
-    val primaryLang: Option[ProgrammingLang] = None,
-    val langsStat: Option[Seq[ProgrammingLangStat]] = None) {
+case class Repo(
+    name: String,
+    owner: Account,
+    description: Option[String] = None,
+    isPrivate: Option[Boolean] = None,
+    primaryLang: Option[ProgrammingLang] = None,
+    langsStat: Option[Seq[ProgrammingLangStat]] = None) {
   def fullName = s"${owner.name}/$name"
   lazy val langsStatIndex = langsStat.map(seq => seq.map{stat => (stat.lang.code, stat)}.toMap)
   def url: String = owner.getRepoUrl(this)
 }
 
 
 object Repo {
-  //FIXME
-  def getCurrentRepos: Seq[Repo] = List()
 
-  implicit val writer = new Writes[Repo] {
-    def writes(r: Repo) : JsValue = {
+  implicit val repoWrites = new Writes[Repo] {
+    def writes(r: Repo): JsValue = {
       //FIXME: as documented
       Json.obj(
         "name" -> r.name,

app/hedgehog/models/package.scala

@@ -7,4 +7,5 @@ package hedgehog
 package object models {
   val GITHUB_HOST = "github.com"
   val GITHUB_BASE_WEB_URL = s"https://$GITHUB_HOST"
+  val GITHUB_BASE_API_URL = s"https://api.$GITHUB_HOST"
 }

app/hedgehog/models/programming_lang.scala

@@ -5,10 +5,10 @@ package hedgehog.models
  * See LICENSE.txt for details.
  *
  */
-class ProgrammingLang(val code: String)
+case class ProgrammingLang(code: String)
 
 
-class ProgrammingLangStat(
-    val lang: ProgrammingLang,
-    val extensions: Seq[String],
-    val bytes: Option[Long])
+case class ProgrammingLangStat(
+    lang: ProgrammingLang,
+    extensions: Seq[String],
+    bytes: Option[Long])

test/clients/github/ConfigSuite.scala

@@ -15,8 +15,8 @@ class ConfigSuite extends FunSuite with Matchers {
   import hedgehog.clients.github.Config
 
   trait SampleConfigs {
-    val config = new Config(apiBaseUrl = "http://base")
-    val configWithPort = new Config(apiBaseUrl = "http://base:8880")
+    val config = new Config(apiBaseUrl = "http://base", accessToken = None)
+    val configWithPort = new Config(apiBaseUrl = "http://base:8880", accessToken = None)
   }
 
   test("apiBaseUrl property") {