Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
  • Loading branch information
sgrimault committed May 2, 2022
1 parent d2c1982 commit c31527e
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 49 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ data class Taxref(
val description: String,

@SerializedName("regne")
val kingdom: String,
val kingdom: String?,

@SerializedName("group2_inpn")
val group: String
val group: String?
)
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ import fr.geonature.datasync.sync.ServerStatus
import fr.geonature.datasync.sync.io.DatasetJsonReader
import fr.geonature.datasync.sync.io.TaxonomyJsonReader
import fr.geonature.datasync.ui.login.LoginActivity
import kotlinx.coroutines.delay
import org.json.JSONObject
import org.tinylog.Logger
import retrofit2.Response
Expand Down Expand Up @@ -727,6 +728,7 @@ class DataSyncWorker @AssistedInject constructor(
Logger.info { "synchronize taxa..." }

var hasNext: Boolean
var hasError = false
var offset = 0

val validTaxaIds = mutableSetOf<Long>()
Expand All @@ -743,13 +745,15 @@ class DataSyncWorker @AssistedInject constructor(
.awaitResponse()
}.getOrNull()


if (taxrefResponse == null || checkResponse(taxrefResponse).state == WorkInfo.State.FAILED) {
hasNext = false
hasError = true
continue
}

val taxref = runCatching { taxrefResponse.body() }.getOrDefault(emptyList())
val taxref = runCatching { taxrefResponse.body() }
.onFailure { hasError = true }
.getOrDefault(emptyList())

if (taxref?.isEmpty() != false) {
hasNext = false
Expand All @@ -759,21 +763,30 @@ class DataSyncWorker @AssistedInject constructor(
val taxa = taxref
.asSequence()
.map { taxRef ->
Taxon(taxRef.id,
taxRef.name.trim(),
// FIXME: taxRef.kingdom or taxRef.group may be null...
Taxonomy(
// check if this taxon as a valid taxonomy definition
if (taxRef.kingdom.isNullOrBlank() || taxRef.group.isNullOrBlank()) {
Logger.warn { "invalid taxon with ID '${taxRef.id}' found: no taxonomy defined" }

return@map null
}

Taxon(
id = taxRef.id,
name = taxRef.name.trim(),
taxonomy = Taxonomy(
taxRef.kingdom,
taxRef.group
),
taxRef.commonName?.trim(),
taxRef.fullName.trim(),
".+\\[(\\w+) - \\d+]"
commonName = taxRef.commonName?.trim(),
description = taxRef.fullName.trim(),
rank = ".+\\[(\\w+) - \\d+]"
.toRegex()
.find(taxRef.description)?.groupValues
?.elementAtOrNull(1)
?.let { "${it.uppercase(Locale.ROOT)} - ${taxRef.id}" })
?.let { "${it.uppercase(Locale.ROOT)} - ${taxRef.id}" },
)
}
.filterNotNull()
.onEach {
validTaxaIds.add(it.id)
}
Expand Down Expand Up @@ -803,52 +816,63 @@ class DataSyncWorker @AssistedInject constructor(
hasNext = taxref.size == pageSize
} while (hasNext)

setProgress(workData(applicationContext.getString(R.string.sync_data_taxa_orphaned_deleting)))
sendNotification(applicationContext.getString(R.string.sync_data_taxa_orphaned_deleting))

// delete orphaned taxa
val orphanedTaxaIds = mutableSetOf<Long>()
taxonDao
.QB()
.cursor()
.run {
Logger.info { "deleting orphaned taxa..." }
moveToFirst()

while (!isAfterLast) {
Taxon
.fromCursor(this)
?.run {
if (!validTaxaIds.contains(id)) {
orphanedTaxaIds.add(id)
if (hasError) {
Logger.warn { "taxa synchronization finished with errors" }

setProgress(workData(applicationContext.getString(R.string.sync_data_taxa_with_errors)))
sendNotification(applicationContext.getString(R.string.sync_data_taxa_with_errors))

delay(1000)
}

// delete all orphaned if all taxa were synchronized successfully
if (!hasError) {
setProgress(workData(applicationContext.getString(R.string.sync_data_taxa_orphaned_deleting)))
sendNotification(applicationContext.getString(R.string.sync_data_taxa_orphaned_deleting))

val orphanedTaxaIds = mutableSetOf<Long>()
taxonDao
.QB()
.cursor()
.run {
Logger.info { "deleting orphaned taxa..." }
moveToFirst()

while (!isAfterLast) {
Taxon
.fromCursor(this)
?.run {
if (!validTaxaIds.contains(id)) {
orphanedTaxaIds.add(id)
}
}
}

moveToNext()
moveToNext()
}
}
orphanedTaxaIds.forEach {
taxonDao.deleteById(it)
}
orphanedTaxaIds.forEach {
taxonDao.deleteById(it)
}

Logger.info { "orphaned taxa deleted: ${orphanedTaxaIds.size}" }
Logger.info { "orphaned taxa deleted: ${orphanedTaxaIds.size}" }

setProgress(
workData(
setProgress(
workData(
applicationContext.resources.getQuantityString(
R.plurals.sync_data_taxa_orphaned_deleted,
orphanedTaxaIds.size,
orphanedTaxaIds.size
)
)
)
sendNotification(
applicationContext.resources.getQuantityString(
R.plurals.sync_data_taxa_orphaned_deleted,
orphanedTaxaIds.size,
orphanedTaxaIds.size
)
)
)
sendNotification(
applicationContext.resources.getQuantityString(
R.plurals.sync_data_taxa_orphaned_deleted,
orphanedTaxaIds.size,
orphanedTaxaIds.size
)
)
}

if (withAdditionalData) {
Logger.info { "synchronize taxa additional data..." }
Expand All @@ -869,11 +893,13 @@ class DataSyncWorker @AssistedInject constructor(

if (taxrefAreasResponse == null || checkResponse(taxrefAreasResponse).state == WorkInfo.State.FAILED) {
hasNext = false
hasError = true
continue
}

val taxrefAreas =
runCatching { taxrefAreasResponse.body() }.getOrDefault(emptyList())
val taxrefAreas = runCatching { taxrefAreasResponse.body() }
.onFailure { hasError = true }
.getOrDefault(emptyList())

if (taxrefAreas?.isEmpty() != false) {
hasNext = false
Expand Down Expand Up @@ -922,6 +948,15 @@ class DataSyncWorker @AssistedInject constructor(
} while (hasNext)
}

if (hasError) {
Logger.warn { "taxa by area synchronization finished with errors" }

setProgress(workData(applicationContext.getString(R.string.sync_data_taxa_areas_with_errors)))
sendNotification(applicationContext.getString(R.string.sync_data_taxa_areas_with_errors))

delay(1000)
}

return Result.success()
}

Expand Down
2 changes: 2 additions & 0 deletions datasync/src/main/res/values-fr/strings.xml
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,15 @@
<string name="sync_data_nomenclature_default">Synchronisation des valeurs par défaut de la nomenclature : %1$d</string>
<string name="sync_data_nomenclature_default_error">Synchronisation des valeurs par défaut de la nomenclature : échec</string>
<string name="sync_data_taxa">Synchronisation des taxons : %1$d</string>
<string name="sync_data_taxa_with_errors">Synchronisation des taxons terminée avec des erreurs</string>
<string name="sync_data_taxa_orphaned_deleting">Suppression des taxons orphelins…</string>
<plurals name="sync_data_taxa_orphaned_deleted">
<item quantity="one">%d taxon orphelin supprimé</item>
<item quantity="other">%d taxons orphelins supprimés</item>
<item quantity="many">%d taxons orphelins supprimés</item>
</plurals>
<string name="sync_data_taxa_areas">Synchronisation des unités géographiques : %1$d</string>
<string name="sync_data_taxa_areas_with_errors">Synchronisation des unités géographiques terminée avec des erreurs</string>
<string name="sync_data_succeeded">Terminée.</string>

<string name="notification_data_synchronization_title">Synchronisation des données…</string>
Expand Down
4 changes: 3 additions & 1 deletion datasync/src/main/res/values/strings.xml
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,14 @@
<string name="sync_data_nomenclature_default">Synchronize nomenclature default values: %1$d</string>
<string name="sync_data_nomenclature_default_error">Synchronize nomenclature default values: failure</string>
<string name="sync_data_taxa">Synchronize taxa: %1$d</string>
<string name="sync_data_taxa_orphaned_deleting">Deleting orphaned taxa...</string>
<string name="sync_data_taxa_with_errors">Taxa synchronization finished with errors</string>
<string name="sync_data_taxa_orphaned_deleting">Deleting orphaned taxa…</string>
<plurals name="sync_data_taxa_orphaned_deleted">
<item quantity="one">%d orphaned taxon deleted</item>
<item quantity="other">%d orphaned taxa deleted</item>
</plurals>
<string name="sync_data_taxa_areas">Synchronize taxa by area: %1$d</string>
<string name="sync_data_taxa_areas_with_errors">Taxa by area synchronization finished with errors</string>
<string name="sync_data_succeeded">Done.</string>

<string name="notification_data_synchronization_title">Synchronize data…</string>
Expand Down

0 comments on commit c31527e

Please sign in to comment.