Allow opening a legacy (json) db, and example of migration

gradle/libs.versions.toml

@@ -16,6 +16,8 @@ apollo-runtime = { group = "com.apollographql.apollo", name = "apollo-runtime",
 apollo-mockserver = "com.apollographql.mockserver:apollo-mockserver:0.0.1"
 apollo-ast = { group = "com.apollographql.apollo", name = "apollo-ast", version.ref = "apollo" }
 apollo-compiler = { group = "com.apollographql.apollo", name = "apollo-compiler", version.ref = "apollo" }
+apollo-cache = { group = "com.apollographql.apollo", name = "apollo-normalized-cache", version.ref = "apollo" }
+apollo-cache-sqlite = { group = "com.apollographql.apollo", name = "apollo-normalized-cache-sqlite", version.ref = "apollo" }
 atomicfu-library = { group = "org.jetbrains.kotlinx", name = "atomicfu", version.ref = "atomicfu" }
 kotlin-test = { group = "org.jetbrains.kotlin", name = "kotlin-test" } # the Kotlin plugin resolves the version
 kotlin-test-junit = { group = "org.jetbrains.kotlin", name = "kotlin-test-junit" } # the Kotlin plugin resolves the version

normalized-cache-sqlite-incubating/build.gradle.kts

@@ -31,20 +31,15 @@ android {
 
 
 configure<app.cash.sqldelight.gradle.SqlDelightExtension> {
-  databases.create("JsonDatabase") {
-    packageName = "com.apollographql.cache.normalized.sql.internal.json"
-    schemaOutputDirectory = file("sqldelight/json/schema")
-    srcDirs("src/commonMain/sqldelight/json/")
-  }
   databases.create("BlobDatabase") {
-    packageName = "com.apollographql.cache.normalized.sql.internal.blob"
-    schemaOutputDirectory = file("sqldelight/blob/schema")
-    srcDirs("src/commonMain/sqldelight/blob/")
+    packageName.set("com.apollographql.cache.normalized.sql.internal.blob")
+    schemaOutputDirectory.set(file("sqldelight/blob/schema"))
+    srcDirs.setFrom("src/commonMain/sqldelight/blob/")
   }
   databases.create("Blob2Database") {
-    packageName = "com.apollographql.cache.normalized.sql.internal.blob2"
-    schemaOutputDirectory = file("sqldelight/blob2/schema")
-    srcDirs("src/commonMain/sqldelight/blob2/")
+    packageName.set("com.apollographql.cache.normalized.sql.internal.blob2")
+    schemaOutputDirectory.set(file("sqldelight/blob2/schema"))
+    srcDirs.setFrom("src/commonMain/sqldelight/blob2/")
   }
 }
 

normalized-cache-sqlite-incubating/src/commonMain/sqldelight/blob/migrations/1.sqm

@@ -0,0 +1,7 @@
+-- Version 1 is either the blob schema (do nothing) or the legacy json schema (drop and create)
+DROP TABLE IF EXISTS records;
+
+CREATE TABLE IF NOT EXISTS blobs (
+  key TEXT NOT NULL PRIMARY KEY,
+  blob BLOB NOT NULL
+) WITHOUT ROWID;

tests/migration/build.gradle.kts

@@ -0,0 +1,38 @@
+plugins {
+  alias(libs.plugins.kotlin.multiplatform)
+  alias(libs.plugins.apollo)
+}
+
+kotlin {
+  configureKmp(
+      withJs = false,
+      withWasm = false,
+      withAndroid = false,
+      withApple = AppleTargets.Host,
+  )
+
+  sourceSets {
+    getByName("commonMain") {
+      dependencies {
+        implementation(libs.apollo.runtime)
+        implementation(libs.apollo.cache)
+        implementation(libs.apollo.cache.sqlite)
+        implementation("com.apollographql.cache:normalized-cache-sqlite-incubating")
+      }
+    }
+
+    getByName("commonTest") {
+      dependencies {
+        implementation(libs.apollo.testing.support)
+        implementation(libs.apollo.mockserver)
+        implementation(libs.kotlin.test)
+      }
+    }
+  }
+}
+
+apollo {
+  service("service") {
+    packageName.set("test")
+  }
+}

tests/migration/src/commonMain/graphql/extra.graphqls

@@ -0,0 +1,13 @@
+extend schema
+@link(
+  url: "https://specs.apollo.dev/kotlin_labs/v0.3",
+  import: ["@fieldPolicy", "@typePolicy"]
+)
+
+extend type Query
+@fieldPolicy(forField: "user" keyArgs: "id")
+@fieldPolicy(forField: "users" keyArgs: "ids")
+
+extend type User @typePolicy(keyFields: "id")
+
+extend type Repository @typePolicy(keyFields: "id")

tests/migration/src/commonMain/graphql/operations.graphql

@@ -0,0 +1,60 @@
+query MainQuery($userIds: [ID!]!) {
+  me {
+    id
+    name
+    email
+    admin
+    repositories {
+      ...RepositoryFragment
+    }
+  }
+
+  users(ids: $userIds) {
+    id
+    name
+    email
+    admin
+    repositories {
+      ...RepositoryFragment
+    }
+  }
+
+  repositories(first: 15) {
+    ...RepositoryFragment
+  }
+}
+
+query RepositoryListQuery {
+  repositories(first: 15) {
+    id
+    stars
+    starGazers {
+      id
+      name
+    }
+  }
+}
+
+fragment RepositoryFragment on Repository {
+  id
+  starGazers {
+    id
+  }
+}
+
+query ProjectListQuery {
+  projects {
+    velocity
+    isUrgent
+  }
+}
+
+query MetaProjectListQuery {
+  metaProjects {
+    type {
+      owners {
+        name
+      }
+    }
+  }
+}

tests/migration/src/commonMain/graphql/schema.graphqls

@@ -0,0 +1,39 @@
+type Query {
+  me: User!
+  user(id: ID!): User
+  users(ids: [ID!]!): [User!]!
+  repositories(first: Int, after: String): [Repository!]!
+  projects: [Project!]!
+  metaProjects: [[Project!]!]!
+}
+
+type User {
+  id: ID!
+  name: String!
+  email: String
+  admin: Boolean!
+  repositories: [Repository!]!
+}
+
+type Repository {
+  id: ID!
+  stars: Int!
+  starGazers: [User!]!
+}
+
+type Project {
+  id: ID!
+  name: String!
+  description: String
+  owner: User!
+  collaborators: [User!]!
+  velocity: Int!
+  isUrgent: Boolean!
+  type: ProjectType!
+}
+
+type ProjectType {
+  id: ID!
+  name: String!
+  owners: [User!]!
+}

tests/migration/src/commonTest/kotlin/MigrationTest.kt

@@ -0,0 +1,182 @@
+package test
+
+import com.apollographql.apollo.ApolloClient
+import com.apollographql.apollo.exception.CacheMissException
+import com.apollographql.apollo.mpp.currentTimeMillis
+import com.apollographql.apollo.testing.internal.runTest
+import com.apollographql.cache.normalized.ApolloStore
+import com.apollographql.cache.normalized.FetchPolicy
+import com.apollographql.cache.normalized.api.CacheHeaders
+import com.apollographql.cache.normalized.api.CacheKey
+import com.apollographql.cache.normalized.api.DefaultRecordMerger
+import com.apollographql.cache.normalized.api.Record
+import com.apollographql.cache.normalized.api.RecordValue
+import com.apollographql.cache.normalized.fetchPolicy
+import com.apollographql.cache.normalized.memory.MemoryCacheFactory
+import com.apollographql.cache.normalized.sql.SqlNormalizedCacheFactory
+import com.apollographql.cache.normalized.store
+import com.apollographql.mockserver.MockServer
+import com.apollographql.mockserver.enqueueString
+import okio.use
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertIs
+import com.apollographql.apollo.cache.normalized.ApolloStore as LegacyApolloStore
+import com.apollographql.apollo.cache.normalized.api.CacheKey as LegacyCacheKey
+import com.apollographql.apollo.cache.normalized.api.MemoryCacheFactory as LegacyMemoryCacheFactory
+import com.apollographql.apollo.cache.normalized.api.NormalizedCache as LegacyNormalizedCache
+import com.apollographql.apollo.cache.normalized.api.Record as LegacyRecord
+import com.apollographql.apollo.cache.normalized.api.RecordValue as LegacyRecordValue
+import com.apollographql.apollo.cache.normalized.sql.SqlNormalizedCacheFactory as LegacySqlNormalizedCacheFactory
+import com.apollographql.apollo.cache.normalized.store as legacyStore
+
+// language=JSON
+private val REPOSITORY_LIST_RESPONSE = """
+  {
+    "data": {
+      "repositories": [
+        {
+          "__typename": "Repository",
+          "id": "0",
+          "stars": 10,
+          "starGazers": [
+            {
+              "__typename": "User",
+              "id": "0",
+              "name": "John"
+            },
+            {
+              "__typename": "User",
+              "id": "1",
+              "name": "Jane"
+            }
+          ]
+        }
+      ]
+    }
+  }
+""".trimIndent()
+
+private val REPOSITORY_LIST_DATA = RepositoryListQuery.Data(
+    repositories = listOf(
+        RepositoryListQuery.Repository(
+            id = "0",
+            stars = 10,
+            starGazers = listOf(
+                RepositoryListQuery.StarGazer(id = "0", name = "John", __typename = "User"),
+                RepositoryListQuery.StarGazer(id = "1", name = "Jane", __typename = "User"),
+            ),
+            __typename = "Repository"
+        )
+    )
+)
+
+class MigrationTest {
+  @Test
+  fun canOpenLegacyDb() = runTest {
+    val mockServer = MockServer()
+    val name = "apollo-${currentTimeMillis()}.db"
+
+    // Create a legacy store with some data
+    val legacyStore = LegacyApolloStore(LegacyMemoryCacheFactory().chain(LegacySqlNormalizedCacheFactory(name = name)))
+    ApolloClient.Builder()
+        .serverUrl(mockServer.url())
+        .legacyStore(legacyStore)
+        .build()
+        .use { apolloClient ->
+          mockServer.enqueueString(REPOSITORY_LIST_RESPONSE)
+          apolloClient.query(RepositoryListQuery())
+              .fetchPolicy(FetchPolicy.NetworkOnly)
+              .execute()
+        }
+
+    // Open the legacy store which empties it. Add/read some data to make sure it works.
+    val store = ApolloStore(MemoryCacheFactory().chain(SqlNormalizedCacheFactory(name = name)))
+    ApolloClient.Builder()
+        .serverUrl(mockServer.url())
+        .store(store)
+        .build()
+        .use { apolloClient ->
+          // Expected cache miss: the db has been cleared
+          var response = apolloClient.query(RepositoryListQuery())
+              .fetchPolicy(FetchPolicy.CacheOnly)
+              .execute()
+          assertIs<CacheMissException>(response.exception)
+
+          // Add some data
+          mockServer.enqueueString(REPOSITORY_LIST_RESPONSE)
+          apolloClient.query(RepositoryListQuery())
+              .fetchPolicy(FetchPolicy.NetworkOnly)
+              .execute()
+
+          // Read the data back
+          response = apolloClient.query(RepositoryListQuery())
+              .fetchPolicy(FetchPolicy.CacheOnly)
+              .execute()
+          assertEquals(REPOSITORY_LIST_DATA, response.data)
+
+          // Clean up
+          store.clearAll()
+        }
+  }
+
+  @Test
+  fun migrateDb() = runTest {
+    val mockServer = MockServer()
+    // Create a legacy store with some data
+    val legacyStore = LegacyApolloStore(LegacySqlNormalizedCacheFactory(name = "legacy.db")).also { it.clearAll() }
+    ApolloClient.Builder()
+        .serverUrl(mockServer.url())
+        .legacyStore(legacyStore)
+        .build()
+        .use { apolloClient ->
+          mockServer.enqueueString(REPOSITORY_LIST_RESPONSE)
+          apolloClient.query(RepositoryListQuery())
+              .fetchPolicy(FetchPolicy.NetworkOnly)
+              .execute()
+        }
+
+    // Create a modern store and migrate the legacy data
+    val store = ApolloStore(SqlNormalizedCacheFactory(name = "modern.db")).also { it.clearAll() }
+    store.migrateFrom(legacyStore)
+
+    // Read the data back
+    ApolloClient.Builder()
+        .serverUrl(mockServer.url())
+        .store(store)
+        .build()
+        .use { apolloClient ->
+          val response = apolloClient.query(RepositoryListQuery())
+              .fetchPolicy(FetchPolicy.CacheOnly)
+              .execute()
+          assertEquals(REPOSITORY_LIST_DATA, response.data)
+        }
+  }
+}
+
+private fun ApolloStore.migrateFrom(legacyStore: LegacyApolloStore) {
+  accessCache { cache ->
+    cache.merge(
+        records = legacyStore.accessCache { it.allRecords() }.map { it.toRecord() },
+        cacheHeaders = CacheHeaders.NONE,
+        recordMerger = DefaultRecordMerger,
+    )
+  }
+}
+
+private fun LegacyNormalizedCache.allRecords(): List<LegacyRecord> {
+  return dump().values.fold(emptyList()) { acc, map -> acc + map.values }
+}
+
+private fun LegacyRecord.toRecord(): Record = Record(
+    key = key,
+    fields = fields.mapValues { (_, value) -> value.toRecordValue() },
+    mutationId = mutationId
+)
+
+private fun LegacyRecordValue.toRecordValue(): RecordValue = when (this) {
+  is Map<*, *> -> mapValues { (_, value) -> value.toRecordValue() }
+  is List<*> -> map { it.toRecordValue() }
+  is LegacyCacheKey -> CacheKey(key)
+  else -> this
+}