GT-2601: Download translation files from CDN with SHA-256 verification

app/build.gradle.kts

@@ -46,6 +46,7 @@ android {
             applicationIdSuffix = ".stage"
 
             buildConfigField("String", "MOBILE_CONTENT_API", "\"$URI_MOBILE_CONTENT_API_STAGE\"")
+            buildConfigField("String", "MOBILE_CONTENT_CDN", "\"https://mobilecontent-stage.cru.org\"")
 
             // Facebook
             resValue("string", "facebook_app_id", "448969905944197")
@@ -61,6 +62,7 @@ android {
         }
         named("production") {
             buildConfigField("String", "MOBILE_CONTENT_API", "\"$URI_MOBILE_CONTENT_API_PRODUCTION\"")
+            buildConfigField("String", "MOBILE_CONTENT_CDN", "\"https://mobilecontent.cru.org\"")
 
             // Facebook
             resValue("string", "facebook_app_id", "2236701616451487")

app/src/main/kotlin/org/cru/godtools/dagger/ConfigModule.kt

@@ -4,14 +4,15 @@ import dagger.Module
 import dagger.Provides
 import dagger.hilt.InstallIn
 import dagger.hilt.components.SingletonComponent
-import javax.inject.Named
 import org.cru.godtools.BuildConfig
-import org.cru.godtools.api.ApiModule
+import org.cru.godtools.api.ApiConfig
 
 @Module
 @InstallIn(SingletonComponent::class)
 object ConfigModule {
     @get:Provides
-    @get:Named(ApiModule.MOBILE_CONTENT_API_URL)
-    val mobileContentApiBaseUrl = BuildConfig.MOBILE_CONTENT_API
+    val apiConfig = ApiConfig(
+        mobileContentApiUrl = BuildConfig.MOBILE_CONTENT_API,
+        cdnUrl = BuildConfig.MOBILE_CONTENT_CDN
+    )
 }

gradle/libs.versions.toml

@@ -21,7 +21,7 @@ facebook = "18.2.3"
 facebook-flipper = "0.273.0"
 firebase-crashlytics = "20.0.6"
 firebase-perf = "22.0.5"
-godtoolsShared = "1.3.3-SNAPSHOT"
+godtoolsShared = "1.4.0-SNAPSHOT"
 gtoSupport = "4.5.1-SNAPSHOT"
 kotlin = "2.3.21"
 kotlinCoroutines = "1.11.0"

library/api/src/main/kotlin/org/cru/godtools/api/ApiConfig.kt

@@ -0,0 +1,3 @@
+package org.cru.godtools.api
+
+data class ApiConfig(val mobileContentApiUrl: String, val cdnUrl: String)

library/api/src/main/kotlin/org/cru/godtools/api/ApiModule.kt

@@ -79,19 +79,18 @@ object ApiModule {
         .build()
 
     // region mobile-content-api APIs
-    const val MOBILE_CONTENT_API_URL = "MOBILE_CONTENT_API_BASE_URL"
     private const val MOBILE_CONTENT_API = "MOBILE_CONTENT_API"
     private const val MOBILE_CONTENT_API_AUTHENTICATED = "MOBILE_CONTENT_API_AUTHENTICATED"
 
     @Provides
     @Reusable
     @Named(MOBILE_CONTENT_API)
     fun mobileContentApiRetrofit(
-        @Named(MOBILE_CONTENT_API_URL) baseUrl: String,
+        apiConfig: ApiConfig,
         jsonApiConverter: JsonApiConverter,
         okhttp: OkHttpClient,
     ): Retrofit = Retrofit.Builder()
-        .baseUrl(baseUrl)
+        .baseUrl(apiConfig.mobileContentApiUrl)
         .addConverterFactory(LocaleConverterFactory)
         .addConverterFactory(JsonApiConverterFactory(jsonApiConverter))
         .callFactory(okhttp)
@@ -165,14 +164,16 @@ object ApiModule {
     @Provides
     @Reusable
     fun actionCableScarlet(
-        @Named(MOBILE_CONTENT_API_URL) baseUrl: String,
+        apiConfig: ApiConfig,
         app: Application,
         jsonApi: JsonApiConverter,
         okhttp: OkHttpClient,
         referenceLifecycle: ReferenceLifecycle,
     ) = Scarlet.Builder()
         .forceDefaultPlatform()
-        .webSocketFactory(okhttp.newWebSocketFactory(ActionCableRequestFactory("${baseUrl}cable")))
+        .webSocketFactory(
+            okhttp.newWebSocketFactory(ActionCableRequestFactory("${apiConfig.mobileContentApiUrl}cable"))
+        )
         .addMessageAdapterFactory(
             ActionCableMessageAdapterFactory.Builder()
                 .addMessageAdapterFactory(JsonApiMessageAdapterFactory(jsonApi))
@@ -195,4 +196,13 @@ object ApiModule {
         .callFactory(okhttp)
         .build().create<CampaignFormsApi>()
     // endregion Adobe APIs
+
+    // region CDN APIs
+    @Provides
+    @Reusable
+    fun cdnApi(okhttp: OkHttpClient, apiConfig: ApiConfig): CdnApi = Retrofit.Builder().baseUrl(apiConfig.cdnUrl)
+        .callFactory(okhttp)
+        .build()
+        .create()
+    // endregion CDN APIs
 }

library/api/src/main/kotlin/org/cru/godtools/api/CdnApi.kt

@@ -0,0 +1,13 @@
+package org.cru.godtools.api
+
+import okhttp3.ResponseBody
+import retrofit2.Response
+import retrofit2.http.GET
+import retrofit2.http.Path
+import retrofit2.http.Streaming
+
+interface CdnApi {
+    @Streaming
+    @GET("translations/files/{filename}")
+    suspend fun downloadPublishedFile(@Path("filename") name: String): Response<ResponseBody>
+}

library/download-manager/src/main/kotlin/org/cru/godtools/downloadmanager/GodToolsDownloadManager.kt

@@ -43,12 +43,18 @@ import kotlinx.coroutines.flow.transformLatest
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.sync.withLock
 import kotlinx.coroutines.withContext
+import okhttp3.ResponseBody
+import okio.ByteString.Companion.decodeHex
+import okio.HashingSource.Companion.sha256
+import okio.buffer
+import okio.sink
 import org.ccci.gto.android.common.kotlin.coroutines.MutexMap
 import org.ccci.gto.android.common.kotlin.coroutines.ReadWriteMutex
 import org.ccci.gto.android.common.kotlin.coroutines.flow.EmptyStateFlow
 import org.ccci.gto.android.common.kotlin.coroutines.flow.combineTransformLatest
 import org.ccci.gto.android.common.kotlin.coroutines.withLock
 import org.cru.godtools.api.AttachmentsApi
+import org.cru.godtools.api.CdnApi
 import org.cru.godtools.api.TranslationsApi
 import org.cru.godtools.base.Settings
 import org.cru.godtools.base.ToolFileSystem
@@ -66,6 +72,7 @@ import org.cru.godtools.model.Translation
 import org.cru.godtools.model.TranslationKey
 import org.cru.godtools.shared.tool.parser.ManifestParser
 import org.cru.godtools.shared.tool.parser.ParserResult
+import retrofit2.Response
 
 @VisibleForTesting
 internal const val CLEANUP_DELAY = 30_000L
@@ -74,6 +81,7 @@ internal const val CLEANUP_DELAY = 30_000L
 class GodToolsDownloadManager @VisibleForTesting internal constructor(
     private val attachmentsApi: AttachmentsApi,
     private val attachmentsRepository: AttachmentsRepository,
+    private val cdnApi: CdnApi,
     private val downloadedFilesRepository: DownloadedFilesRepository,
     private val fs: ToolFileSystem,
     private val manifestParser: ManifestParser,
@@ -87,6 +95,7 @@ class GodToolsDownloadManager @VisibleForTesting internal constructor(
     internal constructor(
         attachmentsApi: AttachmentsApi,
         attachmentsRepository: AttachmentsRepository,
+        cdnApi: CdnApi,
         downloadedFilesRepository: DownloadedFilesRepository,
         fs: ToolFileSystem,
         manifestParser: ManifestParser,
@@ -96,6 +105,7 @@ class GodToolsDownloadManager @VisibleForTesting internal constructor(
     ) : this(
         attachmentsApi,
         attachmentsRepository,
+        cdnApi,
         downloadedFilesRepository,
         fs,
         manifestParser,
@@ -280,7 +290,7 @@ class GodToolsDownloadManager @VisibleForTesting internal constructor(
     private suspend fun downloadTranslationFiles(translation: Translation): Boolean = filesystemMutex.read.withLock {
         // download manifest if necessary
         val manifestFileName = translation.manifestFileName ?: return false
-        if (!downloadTranslationFileIfNecessary(manifestFileName)) return false
+        if (!downloadPublishedFileIfNecessary(manifestFileName)) return false
 
         // parse manifest
         val parserResult = manifestParser.parseManifest(
@@ -294,9 +304,14 @@ class GodToolsDownloadManager @VisibleForTesting internal constructor(
         val relatedFiles = manifest.relatedFiles
         val completedFiles = AtomicLong(0)
         val successful = coroutineScope {
-            relatedFiles.map {
+            relatedFiles.map { file ->
                 async {
-                    downloadTranslationFileIfNecessary(it).also {
+                    val src = file.src ?: return@async true
+                    downloadPublishedFileIfNecessary(
+                        fileName = src,
+                        sha256 = file.checksumSha256,
+                        size = file.size?.toLong()
+                    ).also {
                         do {
                             val completed = completedFiles.get()
                             updateProgress(key, completed + 1, relatedFiles.size.toLong())
@@ -309,23 +324,55 @@ class GodToolsDownloadManager @VisibleForTesting internal constructor(
 
         // record the translation as downloaded
         downloadedFilesRepository.insertOrIgnore(DownloadedTranslationFile(translation, manifestFileName))
-        relatedFiles.forEach { downloadedFilesRepository.insertOrIgnore(DownloadedTranslationFile(translation, it)) }
+        relatedFiles.mapNotNull { it.src }.forEach {
+            downloadedFilesRepository.insertOrIgnore(DownloadedTranslationFile(translation, it))
+        }
         translationsRepository.markTranslationDownloaded(translation.id, true)
 
         return true
     }
 
-    private suspend fun downloadTranslationFileIfNecessary(fileName: String): Boolean = filesMutex.withLock(fileName) {
+    private suspend fun downloadPublishedFileIfNecessary(
+        fileName: String,
+        sha256: String? = null,
+        size: Long? = null,
+    ): Boolean = filesMutex[fileName].withLock {
         if (downloadedFilesRepository.findDownloadedFile(fileName) != null) return true
-        try {
-            val body = translationsApi.downloadFile(fileName).takeIf { it.isSuccessful }?.body() ?: return false
-            val downloadedFile = DownloadedFile(fileName)
-            withContext(Dispatchers.IO) { body.byteStream().copyTo(downloadedFile) }
-            downloadedFilesRepository.insertOrIgnore(downloadedFile)
-            return true
-        } catch (e: IOException) {
-            return false
+
+        withContext(ioDispatcher) {
+            downloadPublishedFileFromCdn(fileName, sha256, size) ||
+                downloadPublishedFileFromApi(fileName, sha256, size)
+        }
+    }
+
+    private suspend fun downloadPublishedFileFromCdn(fileName: String, sha256: String?, size: Long?) = try {
+        cdnApi.downloadPublishedFile(fileName).storeFile(fileName, sha256, size)
+    } catch (_: IOException) {
+        false
+    }
+
+    private suspend fun downloadPublishedFileFromApi(fileName: String, sha256: String?, size: Long?) = try {
+        translationsApi.downloadFile(fileName).storeFile(fileName, sha256, size)
+    } catch (_: IOException) {
+        false
+    }
+
+    private suspend fun Response<ResponseBody>.storeFile(fileName: String, sha256: String?, size: Long?): Boolean {
+        val body = body().takeIf { isSuccessful } ?: return false
+
+        val downloadedFile = DownloadedFile(fileName)
+        downloadedFile.bufferedSink().use { sink ->
+            body.source().use { source ->
+                val digest = sha256(source)
+                val bytesWritten = sink.writeAll(digest)
+
+                if (size != null && size != bytesWritten) return false
+                if (sha256 != null && sha256.decodeHex() != digest.hash) return false
+            }
         }
+
+        downloadedFilesRepository.insertOrIgnore(downloadedFile)
+        return true
     }
 
     private suspend fun downloadTranslationZip(translation: Translation) = try {
@@ -450,6 +497,8 @@ class GodToolsDownloadManager @VisibleForTesting internal constructor(
     }
     // endregion Cleanup
 
+    private suspend fun DownloadedFile.bufferedSink() = withContext(Dispatchers.IO) { getFile(fs).sink().buffer() }
+
     private suspend fun InputStream.copyTo(file: DownloadedFile) = withContext(Dispatchers.IO) {
         file.getFile(fs).outputStream().use { copyTo(it) }
     }