Skip to content
This repository has been archived by the owner on Aug 16, 2021. It is now read-only.

Commit

Permalink
Merge pull request #50 from justincase-jp/fix/mavenmetadata-to-jcente…
Browse files Browse the repository at this point in the history 
…r-api

Fix/mavenmetadata to jcenter api
  • Loading branch information
@toliner
toliner authored Jul 15, 2020
2 parents c8b8173 + 8b1d277 commit 105ec7c
Show file tree
Hide file tree
Showing 4 changed files with 86 additions and 54 deletions.
6 changes: 3 additions & 3 deletions buildSrc/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import org.jetbrains.kotlin.config.LanguageVersion
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile

plugins {
kotlin("jvm") version "1.3.60"
`embedded-kotlin`
}

repositories {
Expand All @@ -13,8 +13,8 @@ repositories {

dependencies {
implementation(kotlin("stdlib-jdk8", KotlinVersion.CURRENT.toString()))
implementation("io.ktor:ktor-client-cio:1.2.5")
implementation("io.ktor:ktor-client-auth-jvm:1.2.5")
implementation("io.ktor:ktor-client-cio:1.3.2")
implementation("io.ktor:ktor-client-auth-jvm:1.3.2")
implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.9.+")
// https://mvnrepository.com/artifact/org.apache.commons/commons-exec
implementation("org.apache.commons:commons-exec:1.3")
Expand Down
7 changes: 4 additions & 3 deletions buildSrc/src/main/kotlin/BuildFileGenerator.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ object BuildFileGenerator {
writeText(
`get generated build-gradle-kts file as text`(
cdkModule = module,
cdkVersion = cdkVersion.toString(),
cdkVersion = cdkVersion,
bintrayUser = bintrayUser,
bintrayApiKey = bintrayApiKey,
projectVersion = projectVersion
Expand Down Expand Up @@ -298,7 +298,7 @@ object BuildFileGenerator {

private fun `get generated build-gradle-kts file as text`(
cdkModule: String,
cdkVersion: String,
cdkVersion: Version,
bintrayUser: String,
bintrayApiKey: String,
projectVersion: String
Expand All @@ -324,7 +324,8 @@ object BuildFileGenerator {
implementation("jp.justincase.aws-cdk-kotlin-dsl:dsl-common:$projectVersion")
api("software.amazon.awscdk", "$cdkModule", "$cdkVersion")
implementation("software.amazon.awscdk", "core", "$cdkVersion")
${PackageManager.moduleDependencyMap.getValue(cdkModule).joinToString("\n\t") { "api(project(\":$it\"))" }}
${PackageManager.moduleDependencyMap.getValue(cdkVersion).getValue(cdkModule)
.joinToString("\n\t") { "api(project(\":$it\"))" }}
}
publishing {
Expand Down
119 changes: 71 additions & 48 deletions buildSrc/src/main/kotlin/PackageManager.kt
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,24 @@
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import data.BintrayCreatePackageRequestJson
import data.PomArtifact
import data.ResponseJson
import data.Version
import data.*
import io.ktor.client.HttpClient
import io.ktor.client.engine.cio.CIO
import io.ktor.client.features.HttpTimeout
import io.ktor.client.features.auth.Auth
import io.ktor.client.features.auth.providers.basic
import io.ktor.client.request.get
import io.ktor.client.request.post
import io.ktor.client.response.HttpResponse
import io.ktor.client.statement.HttpResponse
import io.ktor.client.statement.readText
import io.ktor.http.ContentType
import io.ktor.http.HttpStatusCode
import io.ktor.http.content.TextContent
import io.ktor.util.KtorExperimentalAPI
import io.ktor.utils.io.jvm.javaio.toInputStream
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.asFlow
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.toList
import kotlinx.coroutines.io.jvm.javaio.toInputStream
import org.w3c.dom.Node
import org.w3c.dom.NodeList
import javax.xml.parsers.DocumentBuilderFactory
Expand All @@ -32,12 +31,27 @@ object PackageManager {
private const val bintrayApiBaseUrl = "https://api.bintray.com/packages/justincase/aws-cdk-kotlin-dsl"

@UseExperimental(KtorExperimentalAPI::class)
private val client = HttpClient(CIO)
private val client = HttpClient(CIO) {
install(HttpTimeout)
}

private val leastVersion = Version("1.20.0")

val cdkModules: Set<String> by lazy {
val response = runBlocking { client.get<String>(requestUrl) }
println("Start to get list of CDK modules")
println(requestUrl)
val response = runBlocking {
val result = kotlin.runCatching {
client.get<String>(requestUrl)
}
while (result.isFailure) {
result.recover {
client.get(requestUrl)
}
}
result.getOrThrow()
}
println("Completed getting list of CDK modules")
val obj = jacksonObjectMapper().readValue<ResponseJson>(response)
obj.response.docs.filter {
it.ec.containsAll(
Expand All @@ -49,37 +63,37 @@ object PackageManager {
}.map { it.a }.filter { "monocdk" !in it }.toSet()
}

val latestGeneratedCdkVersions: Map<String, Version> by lazy {
val bintrayPackageLatestVersion: Map<String, Version> by lazy {
val job = GlobalScope.async {
cdkModules.asFlow()
.map { module ->
val dslMavenMetadataUrl =
"https://dl.bintray.com/justincase/aws-cdk-kotlin-dsl/jp/justincase/aws-cdk-kotlin-dsl/$module/maven-metadata.xml"
val response = withContext(Dispatchers.IO) {
client.get<HttpResponse>(dslMavenMetadataUrl)
}
if (response.status != HttpStatusCode.OK) {
return@map module to leastVersion
}
val doc = withContext(Dispatchers.IO) {
DocumentBuilderFactory.newInstance().newDocumentBuilder()
.parse(response.content.toInputStream())
}
val lastVersionString = doc.getElementsByTagName("latest").item(0).textContent.split('-')[0]
module to Version(lastVersionString)
}.toList().toMap()
println("Start to get latest package version from bintray")
cdkModules.asFlow().map { module ->
val bintrayVersionApiUrl = "$bintrayApiBaseUrl/$module/versions/_latest"
println(bintrayVersionApiUrl)
val response = withContext(Dispatchers.IO) {
client.get<HttpResponse>(bintrayVersionApiUrl)
}
if (response.status != HttpStatusCode.OK) {
return@map module to leastVersion
}
val versionJson = jacksonObjectMapper().readValue<BintrayVersionJson>(response.readText())
val cdkVersionString = versionJson.name.split('-')[0]
module to Version(cdkVersionString)
}.toList().toMap()
}
println("Completed getting latest package version from bintray")
runBlocking {
job.await()
}
}

val cdkVersions: Map<String, List<Version>> by lazy {
runSuspend {
println("Start to get version list of CDK modules")
cdkModules.asFlow()
.map { module ->
val cdkMavenMetadataUrl =
"https://repo1.maven.org/maven2/software/amazon/awscdk/$module/maven-metadata.xml"
println(cdkMavenMetadataUrl)
val response = withContext(Dispatchers.IO) {
client.get<HttpResponse>(cdkMavenMetadataUrl)
}
Expand All @@ -90,7 +104,10 @@ object PackageManager {
module to doc.getElementsByTagName("versions").item(0).childNodes.asList()
.filter { it.nodeName == "version" }
.map { Version(it.textContent) }
.filter { it > leastVersion }
}.toList().toMap()
}.apply {
println("Completed getting version list of CDK modules")
}
}

Expand Down Expand Up @@ -126,7 +143,7 @@ object PackageManager {

val unhandledCdkVersions: Map<String, List<Version>> by lazy {
cdkVersions.mapValues { pair ->
pair.value.filter { it > latestGeneratedCdkVersions.getValue(pair.key) }
pair.value.filter { it > bintrayPackageLatestVersion.getValue(pair.key) }
}
}

Expand All @@ -144,28 +161,34 @@ object PackageManager {
map
}

val moduleDependencyMap: Map<String, List<String>> by lazy {
val moduleDependencyMap: Map<Version, Map<String, List<String>>> by lazy {
runSuspend {
latestCdkVersions.keys.asFlow().map { module ->
val version = latestCdkVersions.getValue(module).toString()
val targetUrl =
"https://repo1.maven.org/maven2/software/amazon/awscdk/$module/$version/$module-${
version}.pom"
val doc = withContext(Dispatchers.IO) {
val response = client.get<HttpResponse>(targetUrl)

check(response.status == HttpStatusCode.OK) { "${response.status} on accessing $targetUrl" }
DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(response.content.toInputStream())
}
val list = doc.getElementsByTagName("dependency").asList().map { node ->
PomArtifact(
node.childNodes.asList().single { it.nodeName == "groupId" }.textContent,
node.childNodes.asList().single { it.nodeName == "artifactId" }.textContent,
node.childNodes.asList().single { it.nodeName == "version" }.textContent
)
}.filter { it.groupId == "software.amazon.awscdk" }.map { it.artifactId }
module to list
}.toList().toMap()
println("Start to get dependencies of CDK modules")
(unhandledCdkModulesForVersions + modulesForLatestCdkVersions).toList().asFlow().map { (version, modules) ->
println("version: $version, module count: ${modules.size}.")
version to modules.map { module ->
val targetUrl =
"https://repo1.maven.org/maven2/software/amazon/awscdk/$module/$version/$module-${version}.pom"
println(targetUrl)
val doc = withContext(Dispatchers.IO) {
val response = client.get<HttpResponse>(targetUrl)

check(response.status == HttpStatusCode.OK) { "${response.status} on accessing $targetUrl" }
DocumentBuilderFactory.newInstance().newDocumentBuilder()
.parse(response.content.toInputStream())
}
val list = doc.getElementsByTagName("dependency").asList().map { node ->
PomArtifact(
node.childNodes.asList().single { it.nodeName == "groupId" }.textContent,
node.childNodes.asList().single { it.nodeName == "artifactId" }.textContent,
node.childNodes.asList().single { it.nodeName == "version" }.textContent
)
}.filter { it.groupId == "software.amazon.awscdk" }.map { it.artifactId }
module to list
}.toMap()
}.toList().toMap().apply {
println("Completed getting dependencies of CDK modules")
}
}
}

Expand Down
8 changes: 8 additions & 0 deletions buildSrc/src/main/kotlin/data/BintrayVersionJson.kt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package data

import com.fasterxml.jackson.annotation.JsonIgnoreProperties

@JsonIgnoreProperties(ignoreUnknown = true)
data class BintrayVersionJson(
val name: String
)

0 comments on commit 105ec7c

Please sign in to comment.