Skip to content

Commit fabe9fd

Browse files
FreekDBFreekDB
committed
Analyze animals and media items in the database
1 parent afdb23a commit fabe9fd

File tree

5 files changed

+132
-28
lines changed

5 files changed

+132
-28
lines changed

‎backupandrestore/src/main/kotlin/com/github/artemo24/dyrbok/analyzeusage/AnalyzeUsage.kt

Lines changed: 104 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,106 @@
11
package com.github.artemo24.dyrbok.analyzeusage
22

3-
import com.github.artemo24.dyrbok.backupandrestore.DyrBokBackupAndRestore
3+
import com.github.artemo24.dyrbok.backupandrestore.main.DyrBokBackupAndRestore
4+
import com.github.artemo24.dyrbok.backupandrestore.dataclasses.FirestoreObjects
5+
import com.github.artemo24.dyrbok.backupandrestore.utilities.StorageUtilities
46
import java.io.FileInputStream
57
import java.util.Properties
68

79

810
fun main() {
9-
AnalyzeUsage().analyzeUsage()
11+
println()
12+
13+
AnalyzeUsage().analyzeAnimalsAndMediaItems()
14+
15+
// Disabled: AnalyzeUsage().analyzeUsageLogging()
1016
}
1117

1218

1319
class AnalyzeUsage {
14-
fun analyzeUsage() {
15-
println()
20+
fun analyzeAnimalsAndMediaItems() {
21+
val firestoreObjects = readFirestoreObjects()
1622

17-
val backupProperties = Properties()
18-
backupProperties.load(FileInputStream("etc/dyrbok-backup.properties"))
19-
// Later: val photoBucketUrl = backupProperties["photoBucketUrl"].toString()
20-
val jsonFilesDirectory = backupProperties["outputDirectory"].toString()
23+
val animalsByAnimalSpecies = firestoreObjects.pets.groupBy { it.pet_type }
24+
25+
// Validation:
26+
//
27+
// - Each animal ID is unique.
28+
// - todo: Each animal's unique name is unique.
29+
// - todo: Each animal is only registered with one animal species.
30+
// - The animal species (pet_type) should not be empty or equal to "unknown".
31+
//
32+
// - Each media item ID is unique.
33+
// - Each media item URL is unique and can be downloaded.
34+
// - Each media item is linked to exactly one animal.
2135

22-
val backupAndRestore = DyrBokBackupAndRestore()
23-
val firestoreObjects = backupAndRestore.readObjectsFromJsonFiles(inputDirectory = jsonFilesDirectory)
36+
val animalIDsToAnimals = mutableMapOf<String, Any>()
37+
val mediaItemIDsToMediaItems = mutableMapOf<String, Any>()
38+
val mediaItemURLsToMediaItems = mutableMapOf<String, Any>()
39+
40+
animalsByAnimalSpecies.keys.sorted().forEach { animalSpecies ->
41+
println("=== === === === === ===")
42+
println(animalSpecies)
43+
animalsByAnimalSpecies[animalSpecies]
44+
?.sortedBy { it.getNameUnique() }
45+
?.forEach { animal ->
46+
println("- ${animal.getNameUnique()} (species: ${animal.pet_type}, ID: ${animal.pet_id})")
47+
48+
processUniqueField(animalIDsToAnimals, animal, animal.pet_id, objectDescription = "animal", uniqueFieldDescription = "ID")
49+
50+
if (animal.pet_type == "") {
51+
println(">>> Unexpected empty animal species for animal with ID '${animal.pet_id}'.")
52+
}
53+
54+
firestoreObjects.mediaItems.filter { it.pet_id == animal.pet_id }.forEach { mediaItem ->
55+
println(" + media item with URL '${StorageUtilities.getShortMediaItemUrl(mediaItem)}'.")
56+
57+
processUniqueField(
58+
mediaItemIDsToMediaItems,
59+
mediaItem,
60+
mediaItem.media_item_id,
61+
objectDescription = "media item",
62+
uniqueFieldDescription = "ID"
63+
)
64+
65+
val mediaItemUrl = mediaItem.photoUrl()
66+
if (mediaItemUrl != null) {
67+
processUniqueField(
68+
mediaItemURLsToMediaItems,
69+
mediaItem,
70+
mediaItemUrl,
71+
objectDescription = "media item",
72+
uniqueFieldDescription = "URL"
73+
)
74+
75+
// todo: Check whether the media item can be downloaded.
76+
} else {
77+
println(">>> Media item $mediaItem does not have a valid media item URL.")
78+
}
79+
}
80+
}
81+
println()
82+
}
83+
}
84+
85+
private fun processUniqueField(
86+
uniqueFieldsToObjects: MutableMap<String, Any>,
87+
theObject: Any,
88+
uniqueField: String,
89+
objectDescription: String,
90+
uniqueFieldDescription: String
91+
) {
92+
if (uniqueFieldsToObjects.contains(uniqueField)) {
93+
println(">>> Duplicate $objectDescription $uniqueFieldDescription '$uniqueField'.")
94+
println(">>> Previous $objectDescription: ${uniqueFieldsToObjects[uniqueField]}.")
95+
println(">>> Current $objectDescription: ${theObject}.")
96+
} else {
97+
uniqueFieldsToObjects[uniqueField] = theObject
98+
}
99+
}
100+
101+
@Suppress("unused")
102+
fun analyzeUsageLogging() {
103+
val firestoreObjects = readFirestoreObjects()
24104

25105
val animalMap = firestoreObjects.pets.associateBy { it.pet_id }
26106
val usersMap = firestoreObjects.users.associateBy { it.user_id }
@@ -45,10 +125,22 @@ class AnalyzeUsage {
45125
println("${logging.date_time} ${userAlias.padStart(length = 20)} $sourceIndicator - $enhancedMessage")
46126
} else if (logging.message.contains(other = "quiz")) {
47127
println("${logging.date_time} ${userAlias.padStart(length = 20)} - $message")
48-
} // else: anonymize other types of messages.
128+
} else {
129+
// todo: Anonymize other types of messages. Search for all users names in each logging message.
130+
println("${logging.date_time} ${userAlias.padStart(length = 20)} - $message")
131+
}
49132
}
50133

51134
println()
52-
println("(W): photo added from the website.")
135+
println("(W): media item added from the website.")
136+
}
137+
138+
private fun readFirestoreObjects(): FirestoreObjects {
139+
val backupProperties = Properties()
140+
backupProperties.load(FileInputStream("etc/dyrbok-backup.properties"))
141+
val jsonFilesDirectory = backupProperties["outputDirectory"].toString()
142+
// Later: val photoBucketUrl = backupProperties["photoBucketUrl"].toString()
143+
144+
return DyrBokBackupAndRestore().readObjectsFromJsonFiles(inputDirectory = jsonFilesDirectory)
53145
}
54146
}

‎backupandrestore/src/main/kotlin/com/github/artemo24/dyrbok/backupandrestore/dataclasses/MediaItem.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ class MediaItem : FirestoreObject {
3636
var website_media_file_url: String = ""
3737

3838
override fun toString(): String =
39-
"Media item with ID '$media_item_id' referencing a photo of the animal with ID '$pet_id'"
39+
"Media item with ID '$media_item_id' referencing a media item of the animal with ID '$pet_id'"
4040

4141
override fun compareTo(other: FirestoreObject): Int =
4242
if (other is MediaItem) media_item_id.compareTo(other.media_item_id) else 0

‎backupandrestore/src/main/kotlin/com/github/artemo24/dyrbok/backupandrestore/dataclasses/Pet.kt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ class Pet : FirestoreObject {
1616
var visible: Boolean = false
1717
var webpage_url: String = ""
1818

19-
fun getNameUnique(): String = unique_name.ifEmpty { name }
19+
fun getNameUnique(): String =
20+
if (unique_name.isEmpty() || unique_name == "$name-1" || unique_name == "$name-*") name else unique_name
2021

2122
override fun toString(): String =
2223
"Animal ${getNameUnique()} (adoption status: ${adoption_status.lowercase()})"

‎backupandrestore/src/main/kotlin/com/github/artemo24/dyrbok/backupandrestore/main/DyrBokBackupAndRestore.kt

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,9 @@ class DyrBokBackupAndRestore {
228228
}
229229

230230
private fun downloadPhotosFromFileStorage(mediaItems: List<MediaItem>, outputDirectory: String) {
231+
// todo: Use MD5 checksum to download only the new files.
232+
// /home/freek/test/android/FirebaseTest1/app/src/main/java/com/github/freekdb/firebasetest1/model/utilities/MD5.kt
233+
231234
val downloadDirectory = "${outputDirectory}animal-photos/"
232235
File(downloadDirectory).mkdirs()
233236

@@ -254,7 +257,7 @@ class DyrBokBackupAndRestore {
254257
val filename = determineFilename(mediaItem, itemIndex)
255258

256259
if (verboseLevel1) {
257-
val shortPhotoUrl = getShortPhotoUrl(photoUrl, mediaItem)
260+
val shortPhotoUrl = StorageUtilities.getShortMediaItemUrl(mediaItem)
258261
println("Download photo ${itemIndex + 1} of $itemCount with URL '$shortPhotoUrl' to file '$filename'.")
259262
}
260263

@@ -265,19 +268,6 @@ class DyrBokBackupAndRestore {
265268
}
266269
}
267270

268-
private fun getShortPhotoUrl(photoUrl: String, mediaItem: MediaItem): String {
269-
val websitePrefix = "https://www.dierenasielleiden.nl/wp-content/uploads/"
270-
val firebasePrefix = StorageUtilities.getPhotoBucketUrl()
271-
val gitHubPrefix = "https://raw.githubusercontent.com/FreekDB/repository-size-test/main/"
272-
273-
return when {
274-
photoUrl.startsWith(prefix = websitePrefix) -> "website: ${photoUrl.substringAfter(delimiter = websitePrefix)}"
275-
photoUrl.startsWith(prefix = firebasePrefix) -> "Firebase: ${mediaItem.storage_filepath.substringAfterLast(delimiter = "/")}"
276-
photoUrl.startsWith(prefix = gitHubPrefix) -> "GitHub: ${photoUrl.substringAfter(delimiter = gitHubPrefix)}"
277-
else -> photoUrl
278-
}
279-
}
280-
281271
private fun determineFilename(mediaItem: MediaItem, itemIndex: Int): String {
282272
val baseFilename = "animal-photo--${mediaItem.media_item_id}--"
283273

‎backupandrestore/src/main/kotlin/com/github/artemo24/dyrbok/backupandrestore/utilities/StorageUtilities.kt

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package com.github.artemo24.dyrbok.backupandrestore.utilities
22

3+
import com.github.artemo24.dyrbok.backupandrestore.dataclasses.MediaItem
4+
35

46
class StorageUtilities {
57
companion object {
@@ -19,5 +21,24 @@ class StorageUtilities {
1921

2022
return url.replace(" ", "%20")
2123
}
24+
25+
fun getShortMediaItemUrl(mediaItem: MediaItem): String {
26+
val websitePrefix = "https://www.dierenasielleiden.nl/wp-content/uploads/"
27+
val firebasePrefix = getPhotoBucketUrl()
28+
val gitHubPrefix = "https://raw.githubusercontent.com/FreekDB/repository-size-test/main/"
29+
30+
val photoUrl = mediaItem.photoUrl()
31+
32+
return if (photoUrl != null) {
33+
when {
34+
photoUrl.startsWith(prefix = websitePrefix) -> "website: ${photoUrl.substringAfter(delimiter = websitePrefix)}"
35+
photoUrl.startsWith(prefix = firebasePrefix) -> "Firebase: ${mediaItem.storage_filepath.substringAfterLast(delimiter = "/")}"
36+
photoUrl.startsWith(prefix = gitHubPrefix) -> "GitHub: ${photoUrl.substringAfter(delimiter = gitHubPrefix)}"
37+
else -> photoUrl
38+
}
39+
} else {
40+
"null"
41+
}
42+
}
2243
}
2344
}

0 commit comments

Comments
 (0)