split node.slice into node.getByOrigin and node.iterate (#397)

Author: SergNikitin

archaeologist/src/background/suggestAssociations.ts

@@ -7,7 +7,7 @@ export async function suggestAssociations(
   limit?: number
 ): Promise<TNode[]> {
   const beagle = Beagle.fromString(phrase)
-  const iter = storage.node.slice({})
+  const iter = storage.node.iterate()
   const suggested: TNode[] = []
   limit = limit ?? 8
   // FIXME(akindyakov): This is a dirty hack to limit time of search by limiting

archaeologist/src/omnibox/omnibox.ts

@@ -52,7 +52,7 @@ const lookUpAndSuggestFor = lodash.debounce(
     suggest: (suggestResults: browser.Omnibox.SuggestResult[]) => void
   ): Promise<void> => {
     const beagle = Beagle.fromString(text)
-    const iter = storage.node.slice({})
+    const iter = storage.node.iterate()
     const suggestions: browser.Omnibox.SuggestResult[] = []
     for (
       let node = await iter.next();

elementary/src/grid/SearchGrid.tsx

@@ -13,8 +13,7 @@ import { SmallCard } from '../SmallCard'
 import { ShrinkCard } from '../ShrinkCard'
 import { NodeTimeBadge } from '../NodeTimeBadge'
 
-import { TNodeSliceIterator } from 'smuggler-api'
-import type { TNode, StorageApi } from 'smuggler-api'
+import type { TNode, INodeIterator, StorageApi } from 'smuggler-api'
 
 import { log, isAbortError, errorise } from 'armoury'
 
@@ -71,12 +70,12 @@ export const SearchGrid = ({
   storage: StorageApi
 }>) => {
   const [search, setUpSearch] = useState<{
-    iter: TNodeSliceIterator
+    iter: INodeIterator
     beagle: Beagle
   } | null>(null)
   useEffect(() => {
     setUpSearch({
-      iter: storage.node.slice({}),
+      iter: storage.node.iterate(),
       beagle: Beagle.fromString(q || undefined),
     })
   }, [q])
@@ -111,7 +110,7 @@ const SearchGridScroll = ({
   storage,
 }: React.PropsWithChildren<{
   beagle: Beagle
-  iter: TNodeSliceIterator
+  iter: INodeIterator
   onCardClick?: (arg0: TNode) => void
   portable?: boolean
   className?: string

smuggler-api/src/api_datacenter.ts

@@ -32,7 +32,6 @@ import {
 } from './types'
 import type {
   CreateNodeArgs,
-  GetNodeSliceArgs,
   NodeBatchRequestBody,
   CreateEdgeArgs,
   StorageApi,
@@ -227,6 +226,25 @@ async function getNode({
   return node
 }
 
+async function getNodesByOrigin({
+  origin,
+}: {
+  origin: OriginId
+  signal?: AbortSignal
+}): Promise<TNode[]> {
+  const sliceAll: GetNodeSliceArgs = {
+    start_time: 0, // since the beginning of time
+    bucket_time_size: 366 * 24 * 60 * 60,
+    origin,
+  }
+  const ret: TNode[] = []
+  const iter = _getNodesSliceIter(sliceAll)
+  for (let node = await iter.next(); node != null; node = await iter.next()) {
+    ret.push(node)
+  }
+  return ret
+}
+
 async function getNodeBatch(
   req: NodeBatchRequestBody,
   signal?: AbortSignal
@@ -347,19 +365,23 @@ const getNodesSlice: GetNodesSliceFn = async ({
   }
 }
 
+type GetNodeSliceArgs = {
+  start_time?: number
+  origin?: OriginId
+  bucket_time_size?: number
+}
+
 function _getNodesSliceIter({
-  end_time,
   start_time,
-  limit,
   origin,
   bucket_time_size,
 }: GetNodeSliceArgs) {
   return new TNodeSliceIterator(
     getNodesSlice,
     start_time,
-    end_time,
+    undefined,
     bucket_time_size,
-    limit,
+    undefined,
     origin
   )
 }
@@ -782,9 +804,10 @@ export function makeDatacenterStorageApi(): StorageApi {
   return {
     node: {
       get: getNode,
+      getByOrigin: getNodesByOrigin,
       update: updateNode,
       create: createNode,
-      slice: _getNodesSliceIter,
+      iterate: () => _getNodesSliceIter({}),
       delete: deleteNode,
       bulkDelete: bulkDeleteNodes,
       batch: {

smuggler-api/src/node_slice_iterator.ts

@@ -5,6 +5,7 @@ export interface INodeIterator {
   next: () => Promise<Optional<TNode>>
   total: () => number
   exhausted: () => boolean
+  abort: () => void
 }
 
 export type GetNodesSliceFn = ({

smuggler-api/src/steroid/node.ts

@@ -314,10 +314,9 @@ export async function lookupNodes(
     return storage.node.get({ nid: key.nid, signal })
   } else if ('webBookmark' in key) {
     const { id, stableUrl } = genOriginId(key.webBookmark.url)
-    const query = { ...SLICE_ALL, origin: { id } }
-    const iter = storage.node.slice(query)
+    const nodes: TNode[] = await storage.node.getByOrigin({ origin: { id } })
 
-    for (let node = await iter.next(); node != null; node = await iter.next()) {
+    for (const node of nodes) {
       const nodeUrl = node.extattrs?.web?.url
       if (nodeUrl && stabiliseUrlForOriginId(nodeUrl) === stableUrl) {
         return node
@@ -326,34 +325,32 @@ export async function lookupNodes(
     return undefined
   } else if ('webQuote' in key) {
     const { id, stableUrl } = genOriginId(key.webQuote.url)
-    const query = { ...SLICE_ALL, origin: { id } }
-    const iter = storage.node.slice(query)
+    const nodes: TNode[] = await storage.node.getByOrigin({ origin: { id } })
 
-    const nodes: TNode[] = []
-    for (let node = await iter.next(); node != null; node = await iter.next()) {
+    const ret: TNode[] = []
+    for (const node of nodes) {
       if (NodeUtil.isWebQuote(node) && node.extattrs?.web_quote) {
         if (
           stabiliseUrlForOriginId(node.extattrs.web_quote.url) === stableUrl
         ) {
-          nodes.push(node)
+          ret.push(node)
         }
       }
     }
     return nodes
   } else if ('url' in key) {
     const { id, stableUrl } = genOriginId(key.url)
-    const query = { ...SLICE_ALL, origin: { id } }
-    const iter = storage.node.slice(query)
+    const nodes: TNode[] = await storage.node.getByOrigin({ origin: { id } })
 
-    const nodes: TNode[] = []
-    for (let node = await iter.next(); node != null; node = await iter.next()) {
+    const ret: TNode[] = []
+    for (const node of nodes) {
       if (NodeUtil.isWebBookmark(node) && node.extattrs?.web) {
         if (stabiliseUrlForOriginId(node.extattrs.web.url) === stableUrl) {
-          nodes.push(node)
+          ret.push(node)
         }
       }
     }
-    return nodes
+    return ret
   }
 
   throw new Error(`Failed to lookup nodes, unsupported key ${key}`)

smuggler-api/src/storage_api.ts

@@ -1,6 +1,6 @@
 import { MimeType } from 'armoury'
 import type { Optional } from 'armoury'
-import { TNodeSliceIterator } from './node_slice_iterator'
+import { INodeIterator } from './node_slice_iterator'
 import {
   TNode,
   NodePatchRequest,
@@ -39,14 +39,6 @@ export type CreateNodeArgs = {
   created_at?: Date
 }
 
-export type GetNodeSliceArgs = {
-  end_time?: number
-  start_time?: number
-  limit?: number
-  origin?: OriginId
-  bucket_time_size?: number
-}
-
 export type NodeBatchRequestBody = {
   nids: Nid[]
 }
@@ -74,6 +66,13 @@ export type SwitchEdgeStickinessArgs = {
 export type StorageApi = {
   node: {
     get: ({ nid, signal }: { nid: Nid; signal?: AbortSignal }) => Promise<TNode>
+    getByOrigin: ({
+      origin,
+      signal,
+    }: {
+      origin: OriginId
+      signal?: AbortSignal
+    }) => Promise<TNode[]>
     update: (
       args: { nid: Nid } & NodePatchRequest,
       signal?: AbortSignal
@@ -82,7 +81,7 @@ export type StorageApi = {
       args: CreateNodeArgs,
       signal?: AbortSignal
     ) => Promise<NewNodeResponse>
-    slice: (args: GetNodeSliceArgs) => TNodeSliceIterator
+    iterate: () => INodeIterator
     delete: ({
       nid,
       signal,

smuggler-api/src/storage_api_throwing.ts

@@ -14,9 +14,10 @@ export function makeAlwaysThrowingStorageApi(): StorageApi {
   return {
     node: {
       get: throwError,
+      getByOrigin: throwError,
       update: throwError,
       create: throwError,
-      slice: throwError,
+      iterate: throwError,
       delete: throwError,
       bulkDelete: throwError,
       batch: {

truthsayer/src/card/SearchAndConnect.tsx

@@ -1,11 +1,10 @@
-import React, { useContext } from 'react'
+import React from 'react'
 import { Modal, Form } from 'react-bootstrap'
 
 import { SearchGrid } from 'elementary'
 
 import lodash from 'lodash'
 import { MzdGlobalContext } from '../lib/global'
-import { StorageApi } from 'smuggler-api'
 
 type SearchAndConnectJinnModalProps = {
   nid: string