Skip to content

Commit

Permalink
initial App.kt added
Browse files Browse the repository at this point in the history
  • Loading branch information
unrec committed Jan 30, 2023
1 parent 31e7b42 commit 5d91279
Showing 3 changed files with 144 additions and 8 deletions.
35 changes: 30 additions & 5 deletions build.gradle.kts
Original file line number Diff line number Diff line change
@@ -7,12 +7,13 @@ java.sourceCompatibility = JavaVersion.VERSION_11

object Versions {

const val KOTLIN = "1.6.21"
const val KOTLIN = "1.8.0"
const val COROUTINES = "1.6.4"
const val JACKSON = "2.14.0"
}

plugins {
kotlin("jvm") version "1.6.21"
kotlin("jvm") version "1.8.0"
id("maven-publish")
application
}
@@ -23,18 +24,42 @@ dependencies {
testImplementation("org.jetbrains.kotlin:kotlin-test")
testImplementation(group = "io.kotest", name = "kotest-assertions-core-jvm", version = "5.5.1")

implementation("org.jetbrains.kotlinx", "kotlinx-coroutines-core", Versions.COROUTINES)
implementation("ru.gildor.coroutines:kotlin-coroutines-okhttp:1.0")

implementation("com.squareup.okhttp3:okhttp:4.9.0")
implementation("me.tongfei","progressbar","0.9.4")

implementation(group = "com.fasterxml.jackson.module", name = "jackson-module-kotlin", version = Versions.JACKSON)
implementation("com.fasterxml.jackson.module", "jackson-module-kotlin", Versions.JACKSON)
implementation("com.fasterxml.jackson.dataformat", "jackson-dataformat-csv", Versions.JACKSON)
implementation("com.fasterxml.jackson.datatype", "jackson-datatype-jsr310", Versions.JACKSON)
}

application {
mainClass.set("unrec.lastfm.tracks.dumper.AppKt")
mainClass.set("com.unrec.lastfm.tracks.dumper.AppKt")
}

tasks.apply {
tasks {

val fatJar = register<Jar>("fatJar") {
dependsOn.addAll(listOf("compileJava", "compileKotlin", "processResources"))

archiveClassifier.set("standalone")
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
manifest {
attributes(mapOf("Main-Class" to application.mainClass))
}

val sourcesMain = sourceSets.main.get()
val contents = configurations.runtimeClasspath.get()
.map { if (it.isDirectory) it else zipTree(it) } + sourcesMain.output
from(contents)
}

build {
dependsOn(fatJar)
}

test {
useJUnitPlatform()
}
111 changes: 111 additions & 0 deletions src/main/kotlin/com/unrec/lastfm/tracks/dumper/App.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package com.unrec.lastfm.tracks.dumper

import com.fasterxml.jackson.core.JsonGenerator
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.csv.CsvGenerator
import com.fasterxml.jackson.dataformat.csv.CsvMapper
import com.fasterxml.jackson.dataformat.csv.CsvSchema
import com.fasterxml.jackson.module.kotlin.registerKotlinModule
import com.unrec.lastfm.tracks.dumper.model.Track
import com.unrec.lastfm.tracks.dumper.model.UserInfo
import com.unrec.lastfm.tracks.dumper.utils.extractTracks
import com.unrec.lastfm.tracks.dumper.utils.extractUser
import com.unrec.lastfm.tracks.dumper.utils.recentTracksGetRequest
import com.unrec.lastfm.tracks.dumper.utils.userInfoGetRequest
import kotlinx.coroutines.runBlocking
import me.tongfei.progressbar.ProgressBar
import okhttp3.ConnectionPool
import okhttp3.OkHttpClient
import ru.gildor.coroutines.okhttp.await
import java.io.File
import java.time.LocalDate
import java.time.format.DateTimeFormatter
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.TimeUnit
import kotlin.system.exitProcess
import kotlin.system.measureTimeMillis
import kotlin.time.DurationUnit
import kotlin.time.toDuration

fun main(args: Array<String>) {

val measureTimeMillis = measureTimeMillis {

// get user info for total pages value
val userInfoRequest = userInfoGetRequest(baseUrl, args[0], args[1])

val userResponse = client.newCall(userInfoRequest).execute()
if (userResponse.code == 404) {
println("Failed to get data for the '${args[0]}' user")
exitProcess(1)
}

val userInfoResponse = client.newCall(userInfoRequest).execute().body?.string()
val userInfo: UserInfo = mapper.extractUser(userInfoResponse!!)
val totalPages = countPages(userInfo.playCount, pageSize)

println("Starting to load Last.fm data for '${args[0]}' user. Total pages to fetch: $totalPages")

// starting to consume tracks
val map = ConcurrentHashMap<Int, List<Track>>()
val progressBar = ProgressBar("Pages processed:", totalPages.toLong())

runBlocking {
for (page in totalPages downTo 1) {
val request = recentTracksGetRequest(baseUrl, args[0], args[1], page, pageSize)
val response = client.newCall(request).await()
val tracks = mapper.extractTracks(response.body?.string()!!)
map[page] = tracks
progressBar.step()
}
}
progressBar.close()

val tracks = mutableListOf<Track>()
for (page in 1..totalPages) {
tracks.addAll(map[page]!!)
}

println("Tracks loaded = ${tracks.size}")

// save tracks to .csv file
val csvMapper = CsvMapper()
csvMapper.configure(JsonGenerator.Feature.IGNORE_UNKNOWN, true)
csvMapper.configure(CsvGenerator.Feature.ALWAYS_QUOTE_STRINGS, false)

val formatter = DateTimeFormatter.ofPattern("yyyy_MM_dd")
val outputFile = File("${args[0]}_${LocalDate.now().format(formatter)}.csv")
val objectWriter = csvMapper.writerFor(Track::class.java).with(schema)
objectWriter.writeValues(outputFile.bufferedWriter()).writeAll(tracks)
}
println("Total dump time = ${measureTimeMillis.toDuration(DurationUnit.MILLISECONDS)}")
}

private const val baseUrl = "http://ws.audioscrobbler.com/2.0/"
private const val pageSize = 200

private val mapper = ObjectMapper().registerKotlinModule()

private val client = OkHttpClient.Builder()
.connectionPool(ConnectionPool(20, 5, TimeUnit.MINUTES))
.connectTimeout(30, TimeUnit.SECONDS)
.writeTimeout(30, TimeUnit.SECONDS)
.retryOnConnectionFailure(true)
.build()

private val schema: CsvSchema = CsvSchema.builder()
.setColumnSeparator(';')
.disableQuoteChar()
.setUseHeader(true)
.addColumn("date")
.addColumn("artist")
.addColumn("track")
.addColumn("album")
.build()

private fun countPages(total: Int, pageSize: Int) = kotlin.math.ceil(total.toDouble() / pageSize).toInt()





6 changes: 3 additions & 3 deletions src/main/kotlin/com/unrec/lastfm/tracks/dumper/model/Track.kt
Original file line number Diff line number Diff line change
@@ -10,19 +10,19 @@ data class Track(
@field:JsonProperty("track_id")
val trackId: String?,

@field:JsonProperty("track_name")
@field:JsonProperty("track")
val trackName: String,

@field:JsonProperty("artist_id")
val artistId: String?,

@field:JsonProperty("artist_name")
@field:JsonProperty("artist")
val artistName: String,

@field:JsonProperty("album_id")
val albumId: String?,

@field:JsonProperty("album_name")
@field:JsonProperty("album")
val albumName: String,

@field:JsonProperty("url")

0 comments on commit 5d91279

Please sign in to comment.