From 89ac47058da8476d62536164552f4c3d82a5b7e7 Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Wed, 4 Jan 2023 07:51:11 +0000 Subject: [PATCH 01/21] replace StorageApi global var with a make fn --- archaeologist/src/background.ts | 4 +- archaeologist/src/content/App.tsx | 7 +- archaeologist/src/popup/PopUpApp.tsx | 4 +- .../src/{api_cloud.ts => api_datacenter.ts} | 113 +++++++++--------- smuggler-api/src/auth/account.ts | 4 +- smuggler-api/src/auth/knocker.ts | 4 +- smuggler-api/src/index.ts | 2 +- smuggler-api/src/steroid/node.ts | 2 +- truthsayer/src/account/create/Signup.tsx | 4 +- truthsayer/src/auth/Login.js | 4 +- truthsayer/src/auth/Logout.js | 4 +- truthsayer/src/auth/PasswordChange.js | 4 +- truthsayer/src/auth/PasswordRecoverForm.js | 4 +- truthsayer/src/auth/PasswordRecoverRequest.js | 4 +- truthsayer/src/crypto/local.ts | 10 +- truthsayer/src/lib/global.tsx | 4 +- 16 files changed, 93 insertions(+), 85 deletions(-) rename smuggler-api/src/{api_cloud.ts => api_datacenter.ts} (94%) diff --git a/archaeologist/src/background.ts b/archaeologist/src/background.ts index 927cae32..55456592 100644 --- a/archaeologist/src/background.ts +++ b/archaeologist/src/background.ts @@ -27,7 +27,7 @@ import { NodeUtil, TotalUserActivity, ResourceVisit, - smuggler, + makeDatacenterStorageApi, UserExternalPipelineId, NodeCreatedVia, UserExternalPipelineIngestionProgress, @@ -877,7 +877,7 @@ browser.contextMenus.onClicked.addListener( } ) -const storage: StorageApi = smuggler +const storage: StorageApi = makeDatacenterStorageApi() auth.register() browserBookmarks.register(storage) diff --git a/archaeologist/src/content/App.tsx b/archaeologist/src/content/App.tsx index b7213ed6..e8698f6f 100644 --- a/archaeologist/src/content/App.tsx +++ b/archaeologist/src/content/App.tsx @@ -5,7 +5,7 @@ import browser from 'webextension-polyfill' import { PostHog } from 'posthog-js' import { v4 as uuidv4 } from 'uuid' -import { NodeUtil, NodeType, smuggler } from 'smuggler-api' +import { NodeUtil, NodeType, makeDatacenterStorageApi } from 'smuggler-api' import type { TNode, TNodeJson } from 'smuggler-api' import { genOriginId, OriginIdentity, log, productanalytics } from 'armoury' import * as truthsayer_archaeologist_communication from 'truthsayer-archaeologist-communication' @@ -390,7 +390,10 @@ const App = () => { return ( { return ( - + {state.userUid == null ? : } diff --git a/smuggler-api/src/api_cloud.ts b/smuggler-api/src/api_datacenter.ts similarity index 94% rename from smuggler-api/src/api_cloud.ts rename to smuggler-api/src/api_datacenter.ts index 172f51b0..a27fe9ff 100644 --- a/smuggler-api/src/api_cloud.ts +++ b/smuggler-api/src/api_datacenter.ts @@ -1,6 +1,6 @@ /** - * Implementation of smuggler APIs like @see StorageApi which interact with a - * cloud-hosted infrastructure to perform their job. + * Implementation of smuggler APIs like @see StorageApi which, to perform their job, + * interact with an infrastructure hosted in Mazed's datacenter. */ import { @@ -233,7 +233,7 @@ async function lookupNodes(key: NodeLookupKey, signal?: AbortSignal) { } else if ('webBookmark' in key) { const { id, stableUrl } = genOriginId(key.webBookmark.url) const query = { ...SLICE_ALL, origin: { id } } - const iter = smuggler.node.slice(query) + const iter = _getNodesSliceIter(query) for (let node = await iter.next(); node != null; node = await iter.next()) { const nodeUrl = node.extattrs?.web?.url @@ -245,7 +245,7 @@ async function lookupNodes(key: NodeLookupKey, signal?: AbortSignal) { } else if ('webQuote' in key) { const { id, stableUrl } = genOriginId(key.webQuote.url) const query = { ...SLICE_ALL, origin: { id } } - const iter = smuggler.node.slice(query) + const iter = _getNodesSliceIter(query) const nodes: TNode[] = [] for (let node = await iter.next(); node != null; node = await iter.next()) { @@ -261,7 +261,7 @@ async function lookupNodes(key: NodeLookupKey, signal?: AbortSignal) { } else if ('url' in key) { const { id, stableUrl } = genOriginId(key.url) const query = { ...SLICE_ALL, origin: { id } } - const iter = smuggler.node.slice(query) + const iter = _getNodesSliceIter(query) const nodes: TNode[] = [] for (let node = await iter.next(); node != null; node = await iter.next()) { @@ -957,62 +957,67 @@ function _makeResponseError(response: Response, message?: string): Error { }) } -export const smuggler: StorageApi & AuthenticationApi = { - getAuth, - node: { - get: getNode, - update: updateNode, - create: createNode, - createOrUpdate: createOrUpdateNode, - slice: _getNodesSliceIter, - lookup: lookupNodes, - delete: deleteNode, - bulkDelete: bulkDeleteNodes, - batch: { - /** - * Caution: these methods are not cached - */ - get: getNodeBatch, +export function makeDatacenterStorageApi(): StorageApi { + return { + node: { + get: getNode, + update: updateNode, + create: createNode, + createOrUpdate: createOrUpdateNode, + slice: _getNodesSliceIter, + lookup: lookupNodes, + delete: deleteNode, + bulkDelete: bulkDeleteNodes, + batch: { + /** + * Caution: these methods are not cached + */ + get: getNodeBatch, + }, + url: makeDirectUrl, }, - url: makeDirectUrl, - }, - blob: { - upload: uploadFiles, - sourceUrl: makeBlobSourceUrl, - }, - blob_index: { - build: buildFilesSearchIndex, - cfg: { - supportsMime: mimeTypeIsSupportedByBuildIndex, + blob: { + upload: uploadFiles, + sourceUrl: makeBlobSourceUrl, }, - }, - edge: { - create: createEdge, - get: getNodeAllEdges, - sticky: switchEdgeStickiness, - delete: deleteEdge, - }, + blob_index: { + build: buildFilesSearchIndex, + cfg: { + supportsMime: mimeTypeIsSupportedByBuildIndex, + }, + }, + edge: { + create: createEdge, + get: getNodeAllEdges, + sticky: switchEdgeStickiness, + delete: deleteEdge, + }, + activity: { + external: { + add: addExternalUserActivity, + get: getExternalUserActivity, + }, + association: { + record: recordExternalAssociation, + get: getExternalAssociation, + }, + }, + external: { + ingestion: { + get: getUserIngestionProgress, + advance: advanceUserIngestionProgress, + }, + }, + } +} + +export const authentication: AuthenticationApi = { + getAuth, session: { create: createSession, delete: deleteSession, update: updateSession, }, - activity: { - external: { - add: addExternalUserActivity, - get: getExternalUserActivity, - }, - association: { - record: recordExternalAssociation, - get: getExternalAssociation, - }, - }, - external: { - ingestion: { - get: getUserIngestionProgress, - advance: advanceUserIngestionProgress, - }, - }, user: { password: { recover: passwordRecoverRequest, diff --git a/smuggler-api/src/auth/account.ts b/smuggler-api/src/auth/account.ts index ca26ca85..222ebf3f 100644 --- a/smuggler-api/src/auth/account.ts +++ b/smuggler-api/src/auth/account.ts @@ -1,4 +1,4 @@ -import { smuggler } from './../api_cloud' +import { authentication } from '../api_datacenter' import { authCookie } from './cookie' import { AccountInterface } from './account_interface' import type { LocalCrypto } from './account_interface' @@ -40,7 +40,7 @@ export class UserAccount extends AnonymousAccount { } static async create(signal?: AbortSignal): Promise { - const user = await smuggler.getAuth({ signal }).catch(() => { + const user = await authentication.getAuth({ signal }).catch(() => { return null }) if (!user) { diff --git a/smuggler-api/src/auth/knocker.ts b/smuggler-api/src/auth/knocker.ts index 7ef12ab8..c794244d 100644 --- a/smuggler-api/src/auth/knocker.ts +++ b/smuggler-api/src/auth/knocker.ts @@ -1,4 +1,4 @@ -import { smuggler, SmugglerError } from '../api_cloud' +import { authentication, SmugglerError } from '../api_datacenter' import { StatusCode } from './../status_codes' import { authCookie } from './cookie' @@ -106,7 +106,7 @@ export class Knocker { const now = unixtime.now() if (this.#knockingPeriodSeconds < now - lastUpdateTime) { log.debug('Knock-knock smuggler', now, lastUpdateTime) - await smuggler.session.update(this.#abortController.signal) + await authentication.session.update(this.#abortController.signal) this.setLastUpdate({ time: now }) try { diff --git a/smuggler-api/src/index.ts b/smuggler-api/src/index.ts index 3d45f328..a4f6ea59 100644 --- a/smuggler-api/src/index.ts +++ b/smuggler-api/src/index.ts @@ -8,5 +8,5 @@ export * from './node_slice_iterator' export * from './storage_api' export * from './storage_api_throwing' export * from './authentication_api' -export { smuggler } from './api_cloud' +export { authentication, makeDatacenterStorageApi } from './api_datacenter' export { steroid } from './steroid/steroid' diff --git a/smuggler-api/src/steroid/node.ts b/smuggler-api/src/steroid/node.ts index d58a2607..30f1bb09 100644 --- a/smuggler-api/src/steroid/node.ts +++ b/smuggler-api/src/steroid/node.ts @@ -34,7 +34,7 @@ export type CreateNodeFromLocalBinaryArgs = { /** * Upload a local binary file as a *fully featured* Mazed node - * (as opposed to, for example, @see smuggler.blob.upload that + * (as opposed to, for example, @see StorageApi.blob.upload that * at the time of this writing creates a node that *doesn't support some * Mazed features* like search index). */ diff --git a/truthsayer/src/account/create/Signup.tsx b/truthsayer/src/account/create/Signup.tsx index 4bf163b8..5d57c46f 100644 --- a/truthsayer/src/account/create/Signup.tsx +++ b/truthsayer/src/account/create/Signup.tsx @@ -11,7 +11,7 @@ import { goto, History, routes } from '../../lib/route' import { log } from 'armoury' -import { smuggler } from 'smuggler-api' +import { authentication } from 'smuggler-api' import { Link } from 'react-router-dom' type SignupProps = { @@ -92,7 +92,7 @@ class SignupImpl extends React.Component { this.setState({ errorMsg: undefined, }) - smuggler.user + authentication.user .register({ name: this.state.name, email: this.state.email, diff --git a/truthsayer/src/auth/Login.js b/truthsayer/src/auth/Login.js index 6ff66847..66a570c0 100644 --- a/truthsayer/src/auth/Login.js +++ b/truthsayer/src/auth/Login.js @@ -8,7 +8,7 @@ import PropTypes from 'prop-types' import { css } from '@emotion/react' import { withRouter, Link } from 'react-router-dom' -import { smuggler } from 'smuggler-api' +import { authentication } from 'smuggler-api' import { goto } from '../lib/route' class Login extends React.Component { @@ -54,7 +54,7 @@ class Login extends React.Component { }) const { email, password } = this.state const permissions = null - smuggler.session + authentication.session .create( email, password, diff --git a/truthsayer/src/auth/Logout.js b/truthsayer/src/auth/Logout.js index e864f14e..656c8024 100644 --- a/truthsayer/src/auth/Logout.js +++ b/truthsayer/src/auth/Logout.js @@ -4,7 +4,7 @@ import PropTypes from 'prop-types' import { withRouter } from 'react-router-dom' import { goto } from '../lib/route' -import { smuggler } from 'smuggler-api' +import { authentication } from 'smuggler-api' import { MzdGlobalContext } from './../lib/global' @@ -26,7 +26,7 @@ class Logout extends React.Component { const account = this.context.account const isAuthenticated = account != null && account.isAuthenticated() if (isAuthenticated) { - smuggler.session + authentication.session .delete({ signal: this.fetchAbortController.signal, }) diff --git a/truthsayer/src/auth/PasswordChange.js b/truthsayer/src/auth/PasswordChange.js index f1aa9d2d..75184eef 100644 --- a/truthsayer/src/auth/PasswordChange.js +++ b/truthsayer/src/auth/PasswordChange.js @@ -3,7 +3,7 @@ import React from 'react' import { Badge, Button, Card, Col, Container, Form, Row } from 'react-bootstrap' import PropTypes from 'prop-types' -import { smuggler } from 'smuggler-api' +import { authentication } from 'smuggler-api' import { withRouter } from 'react-router-dom' class PasswordChange extends React.Component { @@ -54,7 +54,7 @@ class PasswordChange extends React.Component { onSubmit = (event) => { event.preventDefault() - smuggler.user.password + authentication.user.password .change({ old_password: this.state.password, new_password: this.state.new_password, diff --git a/truthsayer/src/auth/PasswordRecoverForm.js b/truthsayer/src/auth/PasswordRecoverForm.js index 3379a17a..ba93f414 100644 --- a/truthsayer/src/auth/PasswordRecoverForm.js +++ b/truthsayer/src/auth/PasswordRecoverForm.js @@ -5,7 +5,7 @@ import { Card, Button, Form, Container, Row, Col } from 'react-bootstrap' import PropTypes from 'prop-types' import { withRouter } from 'react-router-dom' -import { smuggler } from 'smuggler-api' +import { authentication } from 'smuggler-api' class PasswordRecoverForm extends React.Component { constructor(props) { @@ -36,7 +36,7 @@ class PasswordRecoverForm extends React.Component { onSubmit = (event) => { event.preventDefault() - smuggler.user.password + authentication.user.password .reset({ token: this.props.token, new_password: this.state.password, diff --git a/truthsayer/src/auth/PasswordRecoverRequest.js b/truthsayer/src/auth/PasswordRecoverRequest.js index b07eec1a..d056d2db 100644 --- a/truthsayer/src/auth/PasswordRecoverRequest.js +++ b/truthsayer/src/auth/PasswordRecoverRequest.js @@ -12,7 +12,7 @@ import { import { Emoji } from './../lib/Emoji' import PropTypes from 'prop-types' -import { smuggler } from 'smuggler-api' +import { authentication } from 'smuggler-api' import { withRouter } from 'react-router-dom' class PasswordRecoverRequest extends React.Component { @@ -67,7 +67,7 @@ class PasswordRecoverRequest extends React.Component { onSubmit = (event) => { event.preventDefault() - smuggler.user.password + authentication.user.password .recover({ email: this.state.email, signal: this.abortControler.signal, diff --git a/truthsayer/src/crypto/local.ts b/truthsayer/src/crypto/local.ts index 6b5db12a..7d0c9a67 100644 --- a/truthsayer/src/crypto/local.ts +++ b/truthsayer/src/crypto/local.ts @@ -9,7 +9,7 @@ import { sha1, } from './wrapper' -import { smuggler } from 'smuggler-api' +import { authentication } from 'smuggler-api' import { base64 } from 'armoury' @@ -32,7 +32,7 @@ export class LocalCrypto { _uid: string = '' _storage: TStorage _lastSecret: TSecret | null = null - _smuggler: any + _authentication: any constructor( uid: string, @@ -42,7 +42,7 @@ export class LocalCrypto { ) { this._uid = uid this._storage = storage || ls - this._smuggler = remote || smuggler + this._authentication = remote || authentication this._lastSecret = lastSecret || null } @@ -164,7 +164,7 @@ export class LocalCrypto { return null } const secretEnc: TEncrypted = base64.toObject(secretBase64) - const secondKeyData = await this._smuggler.getSecondKey({ + const secondKeyData = await this._authentication.getSecondKey({ id: secretEnc.secret_id, }) const secondKey: TSecret = { @@ -177,7 +177,7 @@ export class LocalCrypto { } async _storeSecretToLocalStorage(secretPhrase: string): Promise { - const secondKeyData = await this._smuggler.getAnySecondKey() + const secondKeyData = await this._authentication.getAnySecondKey() // { // key: String, // length: u32, diff --git a/truthsayer/src/lib/global.tsx b/truthsayer/src/lib/global.tsx index 051be916..75f89245 100644 --- a/truthsayer/src/lib/global.tsx +++ b/truthsayer/src/lib/global.tsx @@ -9,7 +9,7 @@ import { jcss } from 'elementary' import { createUserAccount, AccountInterface, - smuggler, + makeDatacenterStorageApi, makeAlwaysThrowingStorageApi, } from 'smuggler-api' import type { StorageApi } from 'smuggler-api' @@ -116,7 +116,7 @@ export class MzdGlobal extends React.Component { }, account: null, analytics: props.analytics, - storage: smuggler, + storage: makeDatacenterStorageApi(), } } From 055e2e7f44b1c62e1868647b31377df990770e35 Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Wed, 4 Jan 2023 09:06:17 +0000 Subject: [PATCH 02/21] mv non-trivial StorageApi methods to steroid --- archaeologist/src/background.ts | 3 +- smuggler-api/src/api_datacenter.ts | 183 +------------- smuggler-api/src/steroid/buildIndex.ts | 2 +- smuggler-api/src/steroid/node.ts | 234 +++++++++++++++++- smuggler-api/src/steroid/steroid.ts | 27 +- smuggler-api/src/storage_api.ts | 42 ---- smuggler-api/src/storage_api_throwing.ts | 2 - .../MicrosoftOfficeOneDriveImporter.tsx | 2 +- 8 files changed, 261 insertions(+), 234 deletions(-) diff --git a/archaeologist/src/background.ts b/archaeologist/src/background.ts index 55456592..341f2df3 100644 --- a/archaeologist/src/background.ts +++ b/archaeologist/src/background.ts @@ -32,6 +32,7 @@ import { NodeCreatedVia, UserExternalPipelineIngestionProgress, StorageApi, + steroid, } from 'smuggler-api' import { isReadyToBeAutoSaved } from './background/pageAutoSaving' @@ -98,7 +99,7 @@ async function requestPageSavedStatus(url: string | undefined) { } let nodes try { - nodes = await storage.node.lookup({ url }) + nodes = await steroid(storage).node.lookup({ url }) } catch (err) { log.debug('Lookup by origin ID failed, consider page as non saved', err) return { unmemorable: false } diff --git a/smuggler-api/src/api_datacenter.ts b/smuggler-api/src/api_datacenter.ts index a27fe9ff..8ab6c3b3 100644 --- a/smuggler-api/src/api_datacenter.ts +++ b/smuggler-api/src/api_datacenter.ts @@ -31,9 +31,6 @@ import { UserExternalPipelineIngestionProgress, } from './types' import type { - UniqueNodeLookupKey, - NonUniqueNodeLookupKey, - NodeLookupKey, CreateNodeArgs, GetNodeSliceArgs, NodeBatchRequestBody, @@ -45,11 +42,10 @@ import { makeUrl } from './api_url' import { TNodeSliceIterator, GetNodesSliceFn } from './node_slice_iterator' -import { genOriginId, Mime, stabiliseUrlForOriginId } from 'armoury' +import { Mime } from 'armoury' import type { Optional } from 'armoury' import { MimeType, log } from 'armoury' -import lodash from 'lodash' import moment from 'moment' import { StatusCode } from './status_codes' import { authCookie } from './auth/cookie' @@ -59,15 +55,6 @@ import { AuthenticationApi } from './authentication_api' const kHeaderCreatedAt = 'x-created-at' const kHeaderLastModified = 'last-modified' -export function isUniqueLookupKey( - key: NodeLookupKey -): key is UniqueNodeLookupKey { - if ('nid' in key || 'webBookmark' in key) { - return true - } - return false -} - async function createNode( { text, @@ -111,172 +98,6 @@ async function createNode( throw _makeResponseError(resp) } -function lookupKeyOf(args: CreateNodeArgs): NodeLookupKey | undefined { - // TODO[snikitin@outlook.com]: This ideally should match with NodeUtil.isWebBookmark(), - // NodeUtil.isWebQuote() etc but unclear how to reliably do so. - if (args.extattrs?.web?.url) { - return { webBookmark: { url: args.extattrs.web.url } } - } else if (args.extattrs?.web_quote?.url) { - return { webQuote: { url: args.extattrs.web_quote.url } } - } - return undefined -} - -async function createOrUpdateNode( - args: CreateNodeArgs, - signal?: AbortSignal -): Promise { - const lookupKey = lookupKeyOf(args) - if (!lookupKey || !isUniqueLookupKey(lookupKey)) { - throw new Error( - `Attempt was made to create a node or, if it exists, update it, ` + - `but the input node used look up key ${JSON.stringify(lookupKey)} ` + - `that is not unique, which makes it impossible to correctly handle the 'update' case` - ) - } - const existingNode: TNode | undefined = await lookupNodes(lookupKey) - - if (!existingNode) { - return createNode(args, signal) - } - - const diff = describeWhatWouldPreventNodeUpdate(args, existingNode) - if (diff) { - throw new Error( - `Failed to update node ${existingNode.nid} because some specified fields ` + - `do not support update:\n${diff}` - ) - } - - const patch: NodePatchRequest = { - text: args.text, - index_text: args.index_text, - } - - await updateNode({ nid: existingNode.nid, ...patch }, signal) - return { nid: existingNode.nid } -} - -/** - * At the time of this writing some datapoints that can be specified at - * node creation can't be modified at node update due to API differences - * (@see CreateNodeArgs and @see UpdateNodeArgs). - * This presents a problem for @see createOrUpdate because some of the datapoints - * caller passed in will be ignored in 'update' case, which is not obvious - * and would be unexpected by the caller. - * As a hack this helper tries to check if these datapoints are actually - * different from node's current state. If they are the same then update will - * not result in anything unexpected. - */ -function describeWhatWouldPreventNodeUpdate(args: CreateNodeArgs, node: TNode) { - let diff = '' - const extattrsFieldsOfLittleConsequence = ['description'] - const updatableExtattrsFields = ['text', 'index_text'] - for (const field in args.extattrs) { - const isTargetField = (v: string) => v === field - const isNonUpdatable = - updatableExtattrsFields.findIndex(isTargetField) === -1 - const isOfLittleConsequence = - extattrsFieldsOfLittleConsequence.findIndex(isTargetField) !== -1 - if (!isNonUpdatable || isOfLittleConsequence) { - continue - } - // @ts-ignore: No index signature with a parameter of type 'string' was found on type 'NodeExtattrs' - const lhs = args.extattrs[field] - // @ts-ignore: No index signature with a parameter of type 'string' was found on type 'NodeExtattrs' - const rhs = node.extattrs[field] - if (!lodash.isEqual(lhs, rhs)) { - diff += - `\n\textattrs.${field} - ` + - `${JSON.stringify(lhs)} vs ${JSON.stringify(rhs)}` - } - } - if (args.ntype !== node.ntype) { - diff += `\n\tntype - ${JSON.stringify(args.ntype)} vs ${JSON.stringify( - node.ntype - )}` - } - // At the time of this writing some datapoints that can be set on - // creation of a node do not get sent back when nodes are later retrieved - // from smuggler. That makes it difficult to verify if values in 'args' differ - // from what's stored on smuggler side or not. A conservative validation - // strategy is used ("if a value is set, treat is as an error") to cut corners. - if (args.from_nid) { - diff += `\n\tfrom_nid - ${args.from_nid} vs (data not exposed via smuggler)` - } - if (args.to_nid) { - diff += `\n\tto_nid - ${args.to_nid} vs (data not exposed via smuggler)` - } - - if (!diff) { - return null - } - - return `[what] - [attempted update arg] vs [existing node value]: ${diff}` -} - -async function lookupNodes( - key: UniqueNodeLookupKey, - signal?: AbortSignal -): Promise -async function lookupNodes( - key: NonUniqueNodeLookupKey, - signal?: AbortSignal -): Promise -async function lookupNodes(key: NodeLookupKey, signal?: AbortSignal) { - const SLICE_ALL = { - start_time: 0, // since the beginning of time - bucket_time_size: 366 * 24 * 60 * 60, - } - if ('nid' in key) { - return getNode({ nid: key.nid, signal }) - } else if ('webBookmark' in key) { - const { id, stableUrl } = genOriginId(key.webBookmark.url) - const query = { ...SLICE_ALL, origin: { id } } - const iter = _getNodesSliceIter(query) - - for (let node = await iter.next(); node != null; node = await iter.next()) { - const nodeUrl = node.extattrs?.web?.url - if (nodeUrl && stabiliseUrlForOriginId(nodeUrl) === stableUrl) { - return node - } - } - return undefined - } else if ('webQuote' in key) { - const { id, stableUrl } = genOriginId(key.webQuote.url) - const query = { ...SLICE_ALL, origin: { id } } - const iter = _getNodesSliceIter(query) - - const nodes: TNode[] = [] - for (let node = await iter.next(); node != null; node = await iter.next()) { - if (NodeUtil.isWebQuote(node) && node.extattrs?.web_quote) { - if ( - stabiliseUrlForOriginId(node.extattrs.web_quote.url) === stableUrl - ) { - nodes.push(node) - } - } - } - return nodes - } else if ('url' in key) { - const { id, stableUrl } = genOriginId(key.url) - const query = { ...SLICE_ALL, origin: { id } } - const iter = _getNodesSliceIter(query) - - const nodes: TNode[] = [] - for (let node = await iter.next(); node != null; node = await iter.next()) { - if (NodeUtil.isWebBookmark(node) && node.extattrs?.web) { - if (stabiliseUrlForOriginId(node.extattrs.web.url) === stableUrl) { - nodes.push(node) - } - } - } - return nodes - } - - throw new Error(`Failed to lookup nodes, unsupported key ${key}`) -} - async function uploadFiles( { files, from_nid, to_nid, createdVia }: BlobUploadRequestArgs, signal?: AbortSignal @@ -963,9 +784,7 @@ export function makeDatacenterStorageApi(): StorageApi { get: getNode, update: updateNode, create: createNode, - createOrUpdate: createOrUpdateNode, slice: _getNodesSliceIter, - lookup: lookupNodes, delete: deleteNode, bulkDelete: bulkDeleteNodes, batch: { diff --git a/smuggler-api/src/steroid/buildIndex.ts b/smuggler-api/src/steroid/buildIndex.ts index 95ed1dc6..79abc09d 100644 --- a/smuggler-api/src/steroid/buildIndex.ts +++ b/smuggler-api/src/steroid/buildIndex.ts @@ -33,7 +33,7 @@ async function readAtMost(file: File, maxChars: number) { /** * Build @see NodeIndexText from a file of any type supported in *Mazed* - * (as opposed to @see smuggler.blob_index.build which is limited to file types + * (as opposed to @see StorageApi.blob_index.build which is limited to file types * supported by *smuggler*) */ export async function nodeIndexFromFile( diff --git a/smuggler-api/src/steroid/node.ts b/smuggler-api/src/steroid/node.ts index 30f1bb09..6881489a 100644 --- a/smuggler-api/src/steroid/node.ts +++ b/smuggler-api/src/steroid/node.ts @@ -4,16 +4,29 @@ import { GenerateBlobIndexResponse, + NewNodeResponse, + Nid, NodeCreatedVia, NodeIndexText, + NodePatchRequest, + TNode, UploadMultipartResponse, } from '../types' -import { log, Mime, isAbortError, errorise } from 'armoury' +import { + log, + Mime, + isAbortError, + errorise, + stabiliseUrlForOriginId, + genOriginId, +} from 'armoury' import type { Optional } from 'armoury' -import { StorageApi } from '../storage_api' +import { CreateNodeArgs, StorageApi } from '../storage_api' +import { NodeUtil } from '../typesutil' +import lodash from 'lodash' -// TODO[snikitin@outlook.com] As functions in this module perform -// generation of a file search index, they share a lot of similarities +// TODO[snikitin@outlook.com] Those functions of this module which perform +// generation of a file search index share a lot of similarities // with @see nodeIndexFromFile(). It may be beneficial if we can reuse one // from another since right now new index-related features have to be implemented // multiple times. @@ -132,3 +145,216 @@ export async function createNodeFromLocalBinary({ return toIndexRelatedWarning(errorise(error)) } } + +export function isUniqueLookupKey( + key: NodeLookupKey +): key is UniqueNodeLookupKey { + if ('nid' in key || 'webBookmark' in key) { + return true + } + return false +} + +function lookupKeyOf(args: CreateNodeArgs): NodeLookupKey | undefined { + // TODO[snikitin@outlook.com]: This ideally should match with NodeUtil.isWebBookmark(), + // NodeUtil.isWebQuote() etc but unclear how to reliably do so. + if (args.extattrs?.web?.url) { + return { webBookmark: { url: args.extattrs.web.url } } + } else if (args.extattrs?.web_quote?.url) { + return { webQuote: { url: args.extattrs.web_quote.url } } + } + return undefined +} + +export async function createOrUpdateNode( + storage: StorageApi, + args: CreateNodeArgs, + signal?: AbortSignal +): Promise { + const lookupKey = lookupKeyOf(args) + if (!lookupKey || !isUniqueLookupKey(lookupKey)) { + throw new Error( + `Attempt was made to create a node or, if it exists, update it, ` + + `but the input node used look up key ${JSON.stringify(lookupKey)} ` + + `that is not unique, which makes it impossible to correctly handle the 'update' case` + ) + } + const existingNode: TNode | undefined = await lookupNodes(storage, lookupKey) + + if (!existingNode) { + return storage.node.create(args, signal) + } + + const diff = describeWhatWouldPreventNodeUpdate(args, existingNode) + if (diff) { + throw new Error( + `Failed to update node ${existingNode.nid} because some specified fields ` + + `do not support update:\n${diff}` + ) + } + + const patch: NodePatchRequest = { + text: args.text, + index_text: args.index_text, + } + + await storage.node.update({ nid: existingNode.nid, ...patch }, signal) + return { nid: existingNode.nid } +} + +/** + * At the time of this writing some datapoints that can be specified at + * node creation can't be modified at node update due to API differences + * (@see CreateNodeArgs and @see UpdateNodeArgs). + * This presents a problem for @see createOrUpdate because some of the datapoints + * caller passed in will be ignored in 'update' case, which is not obvious + * and would be unexpected by the caller. + * As a hack this helper tries to check if these datapoints are actually + * different from node's current state. If they are the same then update will + * not result in anything unexpected. + */ +function describeWhatWouldPreventNodeUpdate(args: CreateNodeArgs, node: TNode) { + let diff = '' + const extattrsFieldsOfLittleConsequence = ['description'] + const updatableExtattrsFields = ['text', 'index_text'] + for (const field in args.extattrs) { + const isTargetField = (v: string) => v === field + const isNonUpdatable = + updatableExtattrsFields.findIndex(isTargetField) === -1 + const isOfLittleConsequence = + extattrsFieldsOfLittleConsequence.findIndex(isTargetField) !== -1 + if (!isNonUpdatable || isOfLittleConsequence) { + continue + } + // @ts-ignore: No index signature with a parameter of type 'string' was found on type 'NodeExtattrs' + const lhs = args.extattrs[field] + // @ts-ignore: No index signature with a parameter of type 'string' was found on type 'NodeExtattrs' + const rhs = node.extattrs[field] + if (!lodash.isEqual(lhs, rhs)) { + diff += + `\n\textattrs.${field} - ` + + `${JSON.stringify(lhs)} vs ${JSON.stringify(rhs)}` + } + } + if (args.ntype !== node.ntype) { + diff += `\n\tntype - ${JSON.stringify(args.ntype)} vs ${JSON.stringify( + node.ntype + )}` + } + // At the time of this writing some datapoints that can be set on + // creation of a node do not get sent back when nodes are later retrieved + // from smuggler. That makes it difficult to verify if values in 'args' differ + // from what's stored on smuggler side or not. A conservative validation + // strategy is used ("if a value is set, treat is as an error") to cut corners. + if (args.from_nid) { + diff += `\n\tfrom_nid - ${args.from_nid} vs (data not exposed via smuggler)` + } + if (args.to_nid) { + diff += `\n\tto_nid - ${args.to_nid} vs (data not exposed via smuggler)` + } + + if (!diff) { + return null + } + + return `[what] - [attempted update arg] vs [existing node value]: ${diff}` +} + +/** + * Unique lookup keys that can match at most 1 node + */ +export type UniqueNodeLookupKey = + /** Due to nid's nature there can be at most 1 node with a particular nid */ + | { nid: Nid } + /** Unique because many nodes can refer to the same URL, but only one of them + * can be a bookmark */ + | { webBookmark: { url: string } } + +export type NonUniqueNodeLookupKey = + /** Can match more than 1 node because multiple parts of a single web page + * can be quoted */ + | { webQuote: { url: string } } + /** Can match more than 1 node because many nodes can refer to + * the same URL: + * - 0 or 1 can be @see NoteType.Url + * - AND at the same time more than 1 can be @see NodeType.WebQuote */ + | { url: string } + +/** + * All the different types of keys that can be used to identify (during lookup, + * for example) one or more nodes. + */ +export type NodeLookupKey = UniqueNodeLookupKey | NonUniqueNodeLookupKey + +export async function lookupNodes( + storage: StorageApi, + key: UniqueNodeLookupKey, + signal?: AbortSignal +): Promise +export async function lookupNodes( + storage: StorageApi, + key: NonUniqueNodeLookupKey, + signal?: AbortSignal +): Promise +export async function lookupNodes( + storage: StorageApi, + key: NodeLookupKey, + signal?: AbortSignal +): Promise +export async function lookupNodes( + storage: StorageApi, + key: NodeLookupKey, + signal?: AbortSignal +): Promise { + const SLICE_ALL = { + start_time: 0, // since the beginning of time + bucket_time_size: 366 * 24 * 60 * 60, + } + if ('nid' in key) { + return storage.node.get({ nid: key.nid, signal }) + } else if ('webBookmark' in key) { + const { id, stableUrl } = genOriginId(key.webBookmark.url) + const query = { ...SLICE_ALL, origin: { id } } + const iter = storage.node.slice(query) + + for (let node = await iter.next(); node != null; node = await iter.next()) { + const nodeUrl = node.extattrs?.web?.url + if (nodeUrl && stabiliseUrlForOriginId(nodeUrl) === stableUrl) { + return node + } + } + return undefined + } else if ('webQuote' in key) { + const { id, stableUrl } = genOriginId(key.webQuote.url) + const query = { ...SLICE_ALL, origin: { id } } + const iter = storage.node.slice(query) + + const nodes: TNode[] = [] + for (let node = await iter.next(); node != null; node = await iter.next()) { + if (NodeUtil.isWebQuote(node) && node.extattrs?.web_quote) { + if ( + stabiliseUrlForOriginId(node.extattrs.web_quote.url) === stableUrl + ) { + nodes.push(node) + } + } + } + return nodes + } else if ('url' in key) { + const { id, stableUrl } = genOriginId(key.url) + const query = { ...SLICE_ALL, origin: { id } } + const iter = storage.node.slice(query) + + const nodes: TNode[] = [] + for (let node = await iter.next(); node != null; node = await iter.next()) { + if (NodeUtil.isWebBookmark(node) && node.extattrs?.web) { + if (stabiliseUrlForOriginId(node.extattrs.web.url) === stableUrl) { + nodes.push(node) + } + } + } + return nodes + } + + throw new Error(`Failed to lookup nodes, unsupported key ${key}`) +} diff --git a/smuggler-api/src/steroid/steroid.ts b/smuggler-api/src/steroid/steroid.ts index 5b32567f..2459be13 100644 --- a/smuggler-api/src/steroid/steroid.ts +++ b/smuggler-api/src/steroid/steroid.ts @@ -12,7 +12,8 @@ * wrapper around individual smuggler's REST API) */ -import { StorageApi } from '../storage_api' +import { CreateNodeArgs, StorageApi } from '../storage_api' +import { TNode } from '../types' import { nodeIndexFromFile, mimeTypeIsSupportedByBuildIndex, @@ -20,9 +21,26 @@ import { import { createNodeFromLocalBinary, CreateNodeFromLocalBinaryArgs, + lookupNodes, + createOrUpdateNode, + UniqueNodeLookupKey, + NonUniqueNodeLookupKey, + NodeLookupKey, } from './node' export const steroid = (storage: StorageApi) => { + const lookup = ((key: NodeLookupKey, signal?: AbortSignal) => + lookupNodes(storage, key, signal)) as { + // See https://stackoverflow.com/a/24222144/3375765 + // and https://stackoverflow.com/a/61366790/3375765 if you are unsure what + // this type signature is + (key: UniqueNodeLookupKey, signal?: AbortSignal): Promise + (key: NonUniqueNodeLookupKey, signal?: AbortSignal): Promise + (key: NodeLookupKey, signal?: AbortSignal): Promise< + TNode[] | TNode | undefined + > + } + return { build_index: { build: (file: File, signal?: AbortSignal) => { @@ -33,6 +51,13 @@ export const steroid = (storage: StorageApi) => { }, }, node: { + createOrUpdate: (args: CreateNodeArgs, signal?: AbortSignal) => + createOrUpdateNode(storage, args, signal), + /** + * Lookup all the nodes that match a given key. For unique lookup keys either + * 0 or 1 nodes will be returned. For non-unique more than 1 node can be returned. + */ + lookup, // TODO[snikitin@outlook.com] See if this can be merged with // uploadLocalTextFile() to form a more general steroid.node.createFromLocal createFromLocalBinary: ( diff --git a/smuggler-api/src/storage_api.ts b/smuggler-api/src/storage_api.ts index 682408af..2b91660c 100644 --- a/smuggler-api/src/storage_api.ts +++ b/smuggler-api/src/storage_api.ts @@ -71,32 +71,6 @@ export type SwitchEdgeStickinessArgs = { signal: AbortSignal } -/** - * Unique lookup keys that can match at most 1 node - */ -export type UniqueNodeLookupKey = - /** Due to nid's nature there can be at most 1 node with a particular nid */ - | { nid: Nid } - /** Unique because many nodes can refer to the same URL, but only one of them - * can be a bookmark */ - | { webBookmark: { url: string } } - -export type NonUniqueNodeLookupKey = - /** Can match more than 1 node because multiple parts of a single web page - * can be quoted */ - | { webQuote: { url: string } } - /** Can match more than 1 node because many nodes can refer to - * the same URL: - * - 0 or 1 can be @see NoteType.Url - * - AND at the same time more than 1 can be @see NodeType.WebQuote */ - | { url: string } - -/** - * All the different types of keys that can be used to identify (during lookup, - * for example) one or more nodes. - */ -export type NodeLookupKey = UniqueNodeLookupKey | NonUniqueNodeLookupKey - export type StorageApi = { node: { get: ({ nid, signal }: { nid: Nid; signal?: AbortSignal }) => Promise @@ -108,23 +82,7 @@ export type StorageApi = { args: CreateNodeArgs, signal?: AbortSignal ) => Promise - createOrUpdate: ( - args: CreateNodeArgs, - signal?: AbortSignal - ) => Promise slice: (args: GetNodeSliceArgs) => TNodeSliceIterator - /** - * Lookup all the nodes that match a given key. For unique lookup keys either - * 0 or 1 nodes will be returned. For non-unique more than 1 node can be returned. - */ - lookup: { - // See https://stackoverflow.com/a/24222144/3375765 if you are unsure what - // this type signature is - (key: UniqueNodeLookupKey, signal?: AbortSignal): Promise< - TNode | undefined - > - (key: NonUniqueNodeLookupKey, signal?: AbortSignal): Promise - } delete: ({ nid, signal, diff --git a/smuggler-api/src/storage_api_throwing.ts b/smuggler-api/src/storage_api_throwing.ts index 18e507fd..391e0cc9 100644 --- a/smuggler-api/src/storage_api_throwing.ts +++ b/smuggler-api/src/storage_api_throwing.ts @@ -16,9 +16,7 @@ export function makeAlwaysThrowingStorageApi(): StorageApi { get: throwError, update: throwError, create: throwError, - createOrUpdate: throwError, slice: throwError, - lookup: throwError, delete: throwError, bulkDelete: throwError, batch: { diff --git a/truthsayer/src/external-import/third-party-filesystem/MicrosoftOfficeOneDriveImporter.tsx b/truthsayer/src/external-import/third-party-filesystem/MicrosoftOfficeOneDriveImporter.tsx index 01e9b29b..63e09fcb 100644 --- a/truthsayer/src/external-import/third-party-filesystem/MicrosoftOfficeOneDriveImporter.tsx +++ b/truthsayer/src/external-import/third-party-filesystem/MicrosoftOfficeOneDriveImporter.tsx @@ -93,7 +93,7 @@ async function uploadFilesFromFolder( created_via: { autoIngestion: epid }, } - const response = await storage.node.createOrUpdate(node) + const response = await steroid(storage).node.createOrUpdate(node) log.debug(`Response to node creation/update: ${JSON.stringify(response)}`) } await storage.external.ingestion.advance(epid, { From ae9c85453abe4de32be3cbebb86973ad5e589999 Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Mon, 9 Jan 2023 09:07:30 +0000 Subject: [PATCH 03/21] implement node.get in local storage --- archaeologist/package.json | 1 + archaeologist/public/manifest.json | 4 +- archaeologist/src/storage_api_local.ts | 276 +++++++++++++++++++++++ smuggler-api/src/types.ts | 15 +- truthsayer/src/card/SmallCardFootbar.tsx | 1 - truthsayer/src/card/Triptych.tsx | 1 - yarn.lock | 17 ++ 7 files changed, 311 insertions(+), 4 deletions(-) create mode 100644 archaeologist/src/storage_api_local.ts diff --git a/archaeologist/package.json b/archaeologist/package.json index 7de2604d..7ae06609 100644 --- a/archaeologist/package.json +++ b/archaeologist/package.json @@ -50,6 +50,7 @@ "@mozilla/readability": "^0.4.1", "@popperjs/core": "^2.10.2", "armoury": "0.0.1", + "base32-encode": "^2.0.0", "bootstrap": "^5.1.3", "css-loader": "^4.3.0", "elementary": "0.0.1", diff --git a/archaeologist/public/manifest.json b/archaeologist/public/manifest.json index 8cfff65a..9d554d92 100644 --- a/archaeologist/public/manifest.json +++ b/archaeologist/public/manifest.json @@ -31,7 +31,9 @@ "cookies", "history", "tabs", - "webNavigation" + "webNavigation", + "storage", + "unlimitedStorage" ], "cross_origin_embedder_policy": { "value": "require-corp" diff --git a/archaeologist/src/storage_api_local.ts b/archaeologist/src/storage_api_local.ts new file mode 100644 index 00000000..15e1c1bb --- /dev/null +++ b/archaeologist/src/storage_api_local.ts @@ -0,0 +1,276 @@ +/** + * An implementation of @see smuggler-api.StorageApi that persists the data + * on user device and is based on browser extension APIs. + * + * It is intended to work in all browser extension contexts (such as background, + * content etc). See https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/storage + * for more information. + */ + +import type { + CreateNodeArgs, + Eid, + NewNodeResponse, + Nid, + NodeCreatedVia, + OriginId, + StorageApi, + TEdgeJson, + TNodeJson, +} from 'smuggler-api' +import { NodeType } from 'smuggler-api' +import { v4 as uuidv4 } from 'uuid' +import base32Encode from 'base32-encode' + +import browser from 'webextension-polyfill' +import { unixtime } from 'armoury' +import lodash from 'lodash' + +// TODO[snikitin@outlook.com] Describe that "yek" is "key" in reverse, +// used to distinguish typesafe keys from just pure string keys used in +// browser.Storage.StorageArea +type GenericYek = { + yek: { + kind: Kind + key: Key + } +} +// TODO[snikitin@outlook.com] Describe that "lav" is "val" (short for "value") +// in reverse, used to distinguish typesafe keys from just "any" JSON value used +// in browser.Storage.StorageArea +type GenericLav = { + lav: { + kind: Kind + value: Value + } +} + +type NidYek = GenericYek<'nid', Nid> +type NidLav = GenericLav<'nid', TNodeJson> + +type OriginToNidYek = GenericYek<'origin->nid', OriginId> +type OriginToNidLav = GenericLav<'origin->nid', Nid[]> + +type NidToEdgeYek = GenericYek<'nid->edge', Nid> +type NidToEdgeLav = GenericLav<'nid->edge', TEdgeJson[]> + +type Yek = NidYek | OriginToNidYek | NidToEdgeYek +type Lav = NidLav | OriginToNidLav | NidToEdgeLav + +// TODO[snikitin@outlook.com] Describe that the purpose of this wrapper is to +// add a bit of ORM-like typesafety to browser.Storage.StorageArea. +// Without this it's very difficult to keep track of what the code is doing +// and what it's supposed to do +class YekLavStore { + private store: browser.Storage.StorageArea + + constructor(store: browser.Storage.StorageArea) { + this.store = store + } + + set(items: { yek: Yek; lav: Lav }[]): Promise { + for (const item of items) { + if (item.yek.yek.kind !== item.lav.lav.kind) { + throw new Error( + `Attempted to set a key/value pair of mismatching kinds: '${item.yek.yek.kind}' !== '${item.lav.lav.kind}'` + ) + } + } + let records: Record = {} + for (const item of items) { + const key = this.stringify(item.yek) + if (records.keys().indexOf(key) !== -1) { + throw new Error(`Attempted to set more than 1 value for key '${key}'`) + } + records[key] = item.lav + } + return this.store.set(records) + } + + get(yek: NidYek): Promise + get(yek: OriginToNidYek): Promise + get(yek: NidToEdgeYek): Promise + get(yek: Yek): Promise { + const key = this.stringify(yek) + const records: Promise> = this.store.get(key) + return records.then( + (records: Record): Promise => { + const record = records[key] + if (record == null) { + return Promise.resolve(undefined) + } + return Promise.resolve(record as Lav) + } + ) + } + + private stringify(yek: Yek): string { + switch (yek.yek.kind) { + case 'nid': + return 'nid:' + yek.yek.key + case 'origin->nid': + return 'origin->nid:' + yek.yek.key + case 'nid->edge': + return 'nid->edge:' + yek.yek.key + } + } +} + +function generateNid(): Nid { + /** + * The implementation tries to produce a @see Nid that's structurally similar + * to the output of smuggler's NodeId::gen(). No attempts are made to mimic + * its behaviour however. + */ + const uuid = uuidv4() + const array: Uint8Array = new TextEncoder().encode(uuid) + return base32Encode(array, 'Crockford') +} + +function generateEid(): Eid { + /** + * The implementation tries to produce a @see Eid that's structurally similar + * to the output of smuggler's EdgeId::gen(). No attempts are made to mimic + * its behaviour however. + */ + return generateNid() +} + +async function createNode( + store: YekLavStore, + args: CreateNodeArgs, + _signal?: AbortSignal +): Promise { + // TODO[snikitin@outlook.com] Below keys must become functional somehow. + // Since they only need to function for search-like actions I can store + // create addition kv-pairs just for them where their value is the key + // (potentially prefixed) and a list of Nid is the value. When a node is added, + // an element gets added into the list and when a node is deleted then an + // element is removed from a list + const origin: OriginId | undefined = args.origin + const created_via: NodeCreatedVia | undefined = args.created_via + + // TODO[snikitin@outlook.com] This graph structure has to work somehow + const from_nid: Nid[] = args.from_nid ?? [] + const to_nid: Nid[] = args.to_nid ?? [] + + const createdAt: number = + args.created_at != null ? unixtime.from(args.created_at) : unixtime.now() + + const node: TNodeJson = { + nid: generateNid(), + ntype: args.ntype ?? NodeType.Text, + text: args.text, + extattrs: args.extattrs, + index_text: args.index_text, + created_at: createdAt, + updated_at: createdAt, + } + + let records: { yek: Yek; lav: Lav }[] = [ + { + yek: { yek: { kind: 'nid', key: node.nid } }, + lav: { lav: { kind: 'nid', value: node } }, + }, + ] + + if (origin) { + const yek: OriginToNidYek = { yek: { kind: 'origin->nid', key: origin } } + let lav: OriginToNidLav | undefined = await store.get(yek) + lav = lav ?? { lav: { kind: 'origin->nid', value: [] } } + const nidsWithThisOrigin: Nid[] = lav.lav.value + nidsWithThisOrigin.push(node.nid) + records.push({ yek, lav }) + } + + if (from_nid.length > 0 || to_nid.length > 0) { + // TODO[snikitin@outlook.com] Evaluate if ownership support is needed + // and implement if yes + const owned_by = 'todo' + + const from_edges: TEdgeJson[] = from_nid.map((from_nid: Nid): TEdgeJson => { + return { + eid: generateEid(), + from_nid, + to_nid: node.nid, + crtd: createdAt, + upd: createdAt, + is_sticky: false, + owned_by, + } + }) + const to_edges: TEdgeJson[] = to_nid.map((to_nid: Nid): TEdgeJson => { + return { + eid: generateEid(), + from_nid: node.nid, + to_nid, + crtd: createdAt, + upd: createdAt, + is_sticky: false, + owned_by, + } + }) + + const yek: NidToEdgeYek = { yek: { kind: 'nid->edge', key: node.nid } } + let lav: NidToEdgeLav = { + lav: { kind: 'nid->edge', value: lodash.concat(from_edges, to_edges) }, + } + // TODO[snikitin@outlook.com] This creates edges for node.nid, but similar + // has to be done for yeks of all the other nids involved + records.push({ yek, lav }) + } + + await store.set(records) + return { nid: node.nid } +} + +export function makeLocalStorageApi( + store: browser.Storage.StorageArea +): StorageApi { + return { + node: { + get: createNode, + // update: updateNode, + // create: createNode, + // slice: _getNodesSliceIter, + // delete: deleteNode, + // bulkDelete: bulkDeleteNodes, + // batch: { + // get: getNodeBatch, + // }, + // url: makeDirectUrl, + }, + blob: { + // upload: uploadFiles, + // sourceUrl: makeBlobSourceUrl, + }, + blob_index: { + // build: buildFilesSearchIndex, + cfg: { + // supportsMime: mimeTypeIsSupportedByBuildIndex, + }, + }, + edge: { + // create: createEdge, + // get: getNodeAllEdges, + // sticky: switchEdgeStickiness, + // delete: deleteEdge, + }, + activity: { + external: { + // add: addExternalUserActivity, + // get: getExternalUserActivity, + }, + association: { + // record: recordExternalAssociation, + // get: getExternalAssociation, + }, + }, + external: { + ingestion: { + // get: getUserIngestionProgress, + // advance: advanceUserIngestionProgress, + }, + }, + } +} diff --git a/smuggler-api/src/types.ts b/smuggler-api/src/types.ts index 368f511a..981d2881 100644 --- a/smuggler-api/src/types.ts +++ b/smuggler-api/src/types.ts @@ -4,6 +4,7 @@ import moment from 'moment' export type SlateText = object[] export type Nid = string +export type Eid = string // Types related to old document types @@ -156,7 +157,7 @@ export type TNode = { } export type TEdge = { - eid: string + eid: Eid txt?: string from_nid: Nid to_nid: Nid @@ -167,6 +168,18 @@ export type TEdge = { owned_by: string } +export type TEdgeJson = { + eid: Eid + txt?: string + from_nid: Nid + to_nid: Nid + crtd: number + upd: number + weight?: number + is_sticky: boolean + owned_by: string +} + /** * Edges of a given node * diff --git a/truthsayer/src/card/SmallCardFootbar.tsx b/truthsayer/src/card/SmallCardFootbar.tsx index 3f44e7b7..078ed40f 100644 --- a/truthsayer/src/card/SmallCardFootbar.tsx +++ b/truthsayer/src/card/SmallCardFootbar.tsx @@ -157,7 +157,6 @@ export function SmallCardFootbar({ cutOffRef, className, }: { - nid: string edge: TEdge switchStickiness: (edge: TEdge, on: boolean) => void cutOffRef: (eid: string) => void diff --git a/truthsayer/src/card/Triptych.tsx b/truthsayer/src/card/Triptych.tsx index 97a1d69c..7a8af2f1 100644 --- a/truthsayer/src/card/Triptych.tsx +++ b/truthsayer/src/card/Triptych.tsx @@ -77,7 +77,6 @@ function RefNodeCard({ `} > Date: Tue, 10 Jan 2023 09:30:59 +0000 Subject: [PATCH 04/21] implement node.* endpoints --- archaeologist/src/storage_api_local.ts | 144 ++++++++++++++++++++----- smuggler-api/src/api_datacenter.ts | 3 +- 2 files changed, 118 insertions(+), 29 deletions(-) diff --git a/archaeologist/src/storage_api_local.ts b/archaeologist/src/storage_api_local.ts index 15e1c1bb..4d85e220 100644 --- a/archaeologist/src/storage_api_local.ts +++ b/archaeologist/src/storage_api_local.ts @@ -7,23 +7,31 @@ * for more information. */ -import type { +import { + Ack, CreateNodeArgs, Eid, + GetNodeSliceArgs, NewNodeResponse, Nid, + NodeBatch, + NodeBatchRequestBody, NodeCreatedVia, + NodePatchRequest, + NodeUtil, OriginId, StorageApi, TEdgeJson, + TNode, TNodeJson, + TNodeSliceIterator, } from 'smuggler-api' import { NodeType } from 'smuggler-api' import { v4 as uuidv4 } from 'uuid' import base32Encode from 'base32-encode' import browser from 'webextension-polyfill' -import { unixtime } from 'armoury' +import { MimeType, unixtime } from 'armoury' import lodash from 'lodash' // TODO[snikitin@outlook.com] Describe that "yek" is "key" in reverse, @@ -90,7 +98,19 @@ class YekLavStore { get(yek: NidYek): Promise get(yek: OriginToNidYek): Promise get(yek: NidToEdgeYek): Promise - get(yek: Yek): Promise { + get(yek: NidYek[]): Promise + get(yek: Yek | Yek[]): Promise { + if (Array.isArray(yek)) { + const keys: string[] = yek.map((value: Yek) => this.stringify(value)) + const records: Promise> = this.store.get(keys) + return records.then((records: Record): Promise => { + const lavs: Lav[] = Object.keys(records).map( + (key: string) => records[key] as Lav + ) + return Promise.resolve(lavs) + }) + } + const key = this.stringify(yek) const records: Promise> = this.store.get(key) return records.then( @@ -138,16 +158,9 @@ function generateEid(): Eid { async function createNode( store: YekLavStore, - args: CreateNodeArgs, - _signal?: AbortSignal + args: CreateNodeArgs ): Promise { // TODO[snikitin@outlook.com] Below keys must become functional somehow. - // Since they only need to function for search-like actions I can store - // create addition kv-pairs just for them where their value is the key - // (potentially prefixed) and a list of Nid is the value. When a node is added, - // an element gets added into the list and when a node is deleted then an - // element is removed from a list - const origin: OriginId | undefined = args.origin const created_via: NodeCreatedVia | undefined = args.created_via // TODO[snikitin@outlook.com] This graph structure has to work somehow @@ -174,8 +187,10 @@ async function createNode( }, ] - if (origin) { - const yek: OriginToNidYek = { yek: { kind: 'origin->nid', key: origin } } + if (args.origin) { + const yek: OriginToNidYek = { + yek: { kind: 'origin->nid', key: args.origin }, + } let lav: OriginToNidLav | undefined = await store.get(yek) lav = lav ?? { lav: { kind: 'origin->nid', value: [] } } const nidsWithThisOrigin: Nid[] = lav.lav.value @@ -224,30 +239,103 @@ async function createNode( return { nid: node.nid } } +async function getNode({ + store, + nid, +}: { + store: YekLavStore + nid: Nid +}): Promise { + const yek: NidYek = { yek: { kind: 'nid', key: nid } } + const lav: NidLav | undefined = await store.get(yek) + if (lav == null) { + throw new Error(`Failed to get node ${nid} because it wasn't found`) + } + const value: TNodeJson = lav.lav.value + return NodeUtil.fromJson(value) +} + +async function getNodeBatch( + store: YekLavStore, + req: NodeBatchRequestBody +): Promise { + const yeks: NidYek[] = req.nids.map((nid: Nid): NidYek => { + return { yek: { kind: 'nid', key: nid } } + }) + const lavs: NidLav[] = await store.get(yeks) + return { nodes: lavs.map((lav: NidLav) => NodeUtil.fromJson(lav.lav.value)) } +} + +async function updateNode( + store: YekLavStore, + args: { nid: Nid } & NodePatchRequest +): Promise { + const yek: NidYek = { yek: { kind: 'nid', key: args.nid } } + const lav: NidLav | undefined = await store.get(yek) + if (lav == null) { + throw new Error(`Failed to update node ${args.nid} because it wasn't found`) + } + const value: TNodeJson = lav.lav.value + value.text = args.text != null ? args.text : value.text + value.index_text = + args.index_text != null ? args.index_text : value.index_text + if (!args.preserve_update_time) { + value.updated_at = unixtime.now() + } + await store.set([{ yek, lav }]) + return { ack: true } +} + export function makeLocalStorageApi( - store: browser.Storage.StorageArea + browserStore: browser.Storage.StorageArea ): StorageApi { + const store = new YekLavStore(browserStore) + + const throwUnimplementedError = (endpoint: string) => { + return (..._: any[]): never => { + throw new Error( + `Attempted to call an ${endpoint} endpoint of local StorageApi which hasn't been implemented yet` + ) + } + } + return { node: { - get: createNode, - // update: updateNode, - // create: createNode, - // slice: _getNodesSliceIter, - // delete: deleteNode, - // bulkDelete: bulkDeleteNodes, - // batch: { - // get: getNodeBatch, - // }, - // url: makeDirectUrl, + get: ({ nid }: { nid: string; signal?: AbortSignal }) => + getNode({ store, nid }), + update: ( + args: { nid: string } & NodePatchRequest, + _signal?: AbortSignal + ) => updateNode(store, args), + create: (args: CreateNodeArgs, _signal?: AbortSignal) => + createNode(store, args), + // TODO[snikitin@outlook.com] Local-hosted slicing implementation is a + // problem because the datacenter-hosted version depends entirely on + // time range search which is easy with in SQL, but with a KV-store + // requires to load all nodes from memory on every "iteration" + slice: throwUnimplementedError('node.slice'), + delete: throwUnimplementedError('node.delete'), + bulkDelete: throwUnimplementedError('node.bulkdDelete'), + batch: { + get: (req: NodeBatchRequestBody, _signal?: AbortSignal) => + getNodeBatch(store, req), + }, + url: throwUnimplementedError('node.url'), }, + // TODO[snikitin@outlook.com] At the time of this writing blob.upload and + // blob_index.build are used together to create a single searchable blob node. + // blob.upload is easy to implement for local storage while blob_index.build + // is more difficult. Given how important search is for Mazed goals at the + // time of writing, it makes little sense to implement any of them until + // blob_index.build is usable. blob: { - // upload: uploadFiles, - // sourceUrl: makeBlobSourceUrl, + upload: throwUnimplementedError('blob.upload'), + sourceUrl: throwUnimplementedError('blob.sourceUrl'), }, blob_index: { - // build: buildFilesSearchIndex, + build: throwUnimplementedError('blob_index.build'), cfg: { - // supportsMime: mimeTypeIsSupportedByBuildIndex, + supportsMime: (_mimeType: MimeType) => false, }, }, edge: { diff --git a/smuggler-api/src/api_datacenter.ts b/smuggler-api/src/api_datacenter.ts index 8ab6c3b3..60749110 100644 --- a/smuggler-api/src/api_datacenter.ts +++ b/smuggler-api/src/api_datacenter.ts @@ -29,6 +29,7 @@ import { UploadMultipartResponse, UserExternalPipelineId, UserExternalPipelineIngestionProgress, + Nid, } from './types' import type { CreateNodeArgs, @@ -247,7 +248,7 @@ async function getNodeBatch( } async function updateNode( - args: { nid: string } & NodePatchRequest, + args: { nid: Nid } & NodePatchRequest, signal?: AbortSignal ): Promise { const { nid, text, index_text, preserve_update_time } = args From 65d7085f2941d29b2d43c97aa0d5c42db37fa25c Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Tue, 10 Jan 2023 09:54:29 +0000 Subject: [PATCH 05/21] implement user activity --- archaeologist/src/storage_api_local.ts | 69 ++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/archaeologist/src/storage_api_local.ts b/archaeologist/src/storage_api_local.ts index 4d85e220..379ba88d 100644 --- a/archaeologist/src/storage_api_local.ts +++ b/archaeologist/src/storage_api_local.ts @@ -9,6 +9,7 @@ import { Ack, + AddUserActivityRequest, CreateNodeArgs, Eid, GetNodeSliceArgs, @@ -25,6 +26,7 @@ import { TNode, TNodeJson, TNodeSliceIterator, + TotalUserActivity, } from 'smuggler-api' import { NodeType } from 'smuggler-api' import { v4 as uuidv4 } from 'uuid' @@ -62,8 +64,11 @@ type OriginToNidLav = GenericLav<'origin->nid', Nid[]> type NidToEdgeYek = GenericYek<'nid->edge', Nid> type NidToEdgeLav = GenericLav<'nid->edge', TEdgeJson[]> -type Yek = NidYek | OriginToNidYek | NidToEdgeYek -type Lav = NidLav | OriginToNidLav | NidToEdgeLav +type OriginToActivityYek = GenericYek<'origin->activity', OriginId> +type OriginToActivityLav = GenericLav<'origin->activity', TotalUserActivity> + +type Yek = NidYek | OriginToNidYek | NidToEdgeYek | OriginToActivityYek +type Lav = NidLav | OriginToNidLav | NidToEdgeLav | OriginToActivityLav // TODO[snikitin@outlook.com] Describe that the purpose of this wrapper is to // add a bit of ORM-like typesafety to browser.Storage.StorageArea. @@ -99,6 +104,7 @@ class YekLavStore { get(yek: OriginToNidYek): Promise get(yek: NidToEdgeYek): Promise get(yek: NidYek[]): Promise + get(yek: OriginToActivityYek): Promise get(yek: Yek | Yek[]): Promise { if (Array.isArray(yek)) { const keys: string[] = yek.map((value: Yek) => this.stringify(value)) @@ -132,6 +138,8 @@ class YekLavStore { return 'origin->nid:' + yek.yek.key case 'nid->edge': return 'nid->edge:' + yek.yek.key + case 'origin->activity': + return 'origin->activity:' + yek.yek.key } } } @@ -286,6 +294,58 @@ async function updateNode( return { ack: true } } +async function addExternalUserActivity( + store: YekLavStore, + origin: OriginId, + activity: AddUserActivityRequest +): Promise { + const total: TotalUserActivity = await getExternalUserActivity(store, origin) + if ('visit' in activity) { + if (activity.visit == null) { + return total + } + total.visits = total.visits.concat(activity.visit.visits) + if (activity.visit.reported_by != null) { + throw new Error( + `activity.external.add does not implement support for visit.reported_by yet` + ) + } + } else if ('attention' in activity) { + if (activity.attention == null) { + return total + } + // TODO[snikitin@outlook.com] What to do with activity.attention.timestamp here? + total.seconds_of_attention += activity.attention.seconds + } + + const yek: OriginToActivityYek = { + yek: { kind: 'origin->activity', key: origin }, + } + const lav: OriginToActivityLav = { + lav: { kind: 'origin->activity', value: total }, + } + await store.set([{ yek, lav }]) + return total +} + +async function getExternalUserActivity( + store: YekLavStore, + origin: OriginId +): Promise { + const yek: OriginToActivityYek = { + yek: { kind: 'origin->activity', key: origin }, + } + const lav: OriginToActivityLav | undefined = await store.get(yek) + if (lav == null) { + return { + visits: [], + seconds_of_attention: 0, + } + } + const value: TotalUserActivity = lav.lav.value + return value +} + export function makeLocalStorageApi( browserStore: browser.Storage.StorageArea ): StorageApi { @@ -346,8 +406,9 @@ export function makeLocalStorageApi( }, activity: { external: { - // add: addExternalUserActivity, - // get: getExternalUserActivity, + add: () => addExternalUserActivity, + get: (origin: OriginId, _signal?: AbortSignal) => + getExternalUserActivity(store, origin), }, association: { // record: recordExternalAssociation, From 6a0759d628310fd6582fbcbb4c78567e9ae73c04 Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Tue, 10 Jan 2023 20:42:50 +0000 Subject: [PATCH 06/21] implement external.ingestion --- archaeologist/src/storage_api_local.ts | 85 +++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 7 deletions(-) diff --git a/archaeologist/src/storage_api_local.ts b/archaeologist/src/storage_api_local.ts index 379ba88d..474536a0 100644 --- a/archaeologist/src/storage_api_local.ts +++ b/archaeologist/src/storage_api_local.ts @@ -10,6 +10,7 @@ import { Ack, AddUserActivityRequest, + AdvanceExternalPipelineIngestionProgress, CreateNodeArgs, Eid, GetNodeSliceArgs, @@ -27,6 +28,8 @@ import { TNodeJson, TNodeSliceIterator, TotalUserActivity, + UserExternalPipelineId, + UserExternalPipelineIngestionProgress, } from 'smuggler-api' import { NodeType } from 'smuggler-api' import { v4 as uuidv4 } from 'uuid' @@ -67,8 +70,24 @@ type NidToEdgeLav = GenericLav<'nid->edge', TEdgeJson[]> type OriginToActivityYek = GenericYek<'origin->activity', OriginId> type OriginToActivityLav = GenericLav<'origin->activity', TotalUserActivity> -type Yek = NidYek | OriginToNidYek | NidToEdgeYek | OriginToActivityYek -type Lav = NidLav | OriginToNidLav | NidToEdgeLav | OriginToActivityLav +type ExtPipelineYek = GenericYek<'ext-pipe', UserExternalPipelineId> +type ExtPipelineLav = GenericLav< + 'ext-pipe', + Omit +> + +type Yek = + | NidYek + | OriginToNidYek + | NidToEdgeYek + | OriginToActivityYek + | ExtPipelineYek +type Lav = + | NidLav + | OriginToNidLav + | NidToEdgeLav + | OriginToActivityLav + | ExtPipelineLav // TODO[snikitin@outlook.com] Describe that the purpose of this wrapper is to // add a bit of ORM-like typesafety to browser.Storage.StorageArea. @@ -105,6 +124,7 @@ class YekLavStore { get(yek: NidToEdgeYek): Promise get(yek: NidYek[]): Promise get(yek: OriginToActivityYek): Promise + get(yek: ExtPipelineYek): Promise get(yek: Yek | Yek[]): Promise { if (Array.isArray(yek)) { const keys: string[] = yek.map((value: Yek) => this.stringify(value)) @@ -135,11 +155,13 @@ class YekLavStore { case 'nid': return 'nid:' + yek.yek.key case 'origin->nid': - return 'origin->nid:' + yek.yek.key + return 'origin->nid:' + yek.yek.key.id case 'nid->edge': return 'nid->edge:' + yek.yek.key case 'origin->activity': - return 'origin->activity:' + yek.yek.key + return 'origin->activity:' + yek.yek.key.id + case 'ext-pipe': + return 'ext-pipe:' + yek.yek.key.pipeline_key } } } @@ -346,6 +368,46 @@ async function getExternalUserActivity( return value } +async function getUserIngestionProgress( + store: YekLavStore, + epid: UserExternalPipelineId +): Promise { + const yek: ExtPipelineYek = { + yek: { kind: 'ext-pipe', key: epid }, + } + const lav: ExtPipelineLav | undefined = await store.get(yek) + if (lav == null) { + return { + epid, + ingested_until: 0, + } + } + const value: UserExternalPipelineIngestionProgress = { + epid, + ...lav.lav.value, + } + return value +} + +async function advanceUserIngestionProgress( + store: YekLavStore, + epid: UserExternalPipelineId, + new_progress: AdvanceExternalPipelineIngestionProgress +): Promise { + const progress: UserExternalPipelineIngestionProgress = + await getUserIngestionProgress(store, epid) + progress.ingested_until = new_progress.ingested_until + + const yek: ExtPipelineYek = { + yek: { kind: 'ext-pipe', key: epid }, + } + const lav: ExtPipelineLav = { + lav: { kind: 'ext-pipe', value: progress }, + } + await store.set([{ yek, lav }]) + return { ack: true } +} + export function makeLocalStorageApi( browserStore: browser.Storage.StorageArea ): StorageApi { @@ -406,7 +468,11 @@ export function makeLocalStorageApi( }, activity: { external: { - add: () => addExternalUserActivity, + add: ( + origin: OriginId, + activity: AddUserActivityRequest, + _signal?: AbortSignal + ) => addExternalUserActivity(store, origin, activity), get: (origin: OriginId, _signal?: AbortSignal) => getExternalUserActivity(store, origin), }, @@ -417,8 +483,13 @@ export function makeLocalStorageApi( }, external: { ingestion: { - // get: getUserIngestionProgress, - // advance: advanceUserIngestionProgress, + get: (epid: UserExternalPipelineId, _signal?: AbortSignal) => + getUserIngestionProgress(store, epid), + advance: ( + epid: UserExternalPipelineId, + new_progress: AdvanceExternalPipelineIngestionProgress, + _signal?: AbortSignal + ) => advanceUserIngestionProgress(store, epid, new_progress), }, }, } From 94d411dd8cf8bda44c8caad501852602bb00bfca Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Wed, 11 Jan 2023 08:31:23 +0000 Subject: [PATCH 07/21] implement edge.* and pass ts checks --- archaeologist/src/storage_api_local.ts | 152 +++++++++++++++++++++++-- smuggler-api/src/storage_api.ts | 4 +- smuggler-api/src/typesutil.ts | 8 ++ 3 files changed, 150 insertions(+), 14 deletions(-) diff --git a/archaeologist/src/storage_api_local.ts b/archaeologist/src/storage_api_local.ts index 474536a0..bbf50301 100644 --- a/archaeologist/src/storage_api_local.ts +++ b/archaeologist/src/storage_api_local.ts @@ -10,8 +10,11 @@ import { Ack, AddUserActivityRequest, + AddUserExternalAssociationRequest, AdvanceExternalPipelineIngestionProgress, + CreateEdgeArgs, CreateNodeArgs, + EdgeUtil, Eid, GetNodeSliceArgs, NewNodeResponse, @@ -19,10 +22,12 @@ import { NodeBatch, NodeBatchRequestBody, NodeCreatedVia, + NodeEdges, NodePatchRequest, NodeUtil, OriginId, StorageApi, + TEdge, TEdgeJson, TNode, TNodeJson, @@ -89,6 +94,12 @@ type Lav = | OriginToActivityLav | ExtPipelineLav +type YekLav = { yek: Yek; lav: Lav } + +function isOfArrayKind(lav: Lav): lav is OriginToNidLav | NidToEdgeLav { + return Array.isArray(lav.lav.value) +} + // TODO[snikitin@outlook.com] Describe that the purpose of this wrapper is to // add a bit of ORM-like typesafety to browser.Storage.StorageArea. // Without this it's very difficult to keep track of what the code is doing @@ -100,7 +111,7 @@ class YekLavStore { this.store = store } - set(items: { yek: Yek; lav: Lav }[]): Promise { + set(items: YekLav[]): Promise { for (const item of items) { if (item.yek.yek.kind !== item.lav.lav.kind) { throw new Error( @@ -125,6 +136,7 @@ class YekLavStore { get(yek: NidYek[]): Promise get(yek: OriginToActivityYek): Promise get(yek: ExtPipelineYek): Promise + get(yek: Yek): Promise get(yek: Yek | Yek[]): Promise { if (Array.isArray(yek)) { const keys: string[] = yek.map((value: Yek) => this.stringify(value)) @@ -150,6 +162,49 @@ class YekLavStore { ) } + // TODO[snikitin@outlook.com] Explain that this method is a poor man's attempt + // to increase atomicity of data insertion + async prepareAppend( + yek: OriginToNidYek, + lav: OriginToNidLav + ): Promise<{ + yek: OriginToNidYek + lav: OriginToNidLav + }> + async prepareAppend( + yek: NidToEdgeYek, + lav: NidToEdgeLav + ): Promise<{ + yek: NidToEdgeYek + lav: NidToEdgeLav + }> + async prepareAppend(yek: Yek, appended_lav: Lav): Promise { + if (yek.yek.kind !== appended_lav.lav.kind) { + throw new Error( + `Attempted to append a key/value pair of mismatching kinds: '${yek.yek.kind}' !== '${appended_lav.lav.kind}'` + ) + } + const lav = await this.get(yek) + if (lav != null && !isOfArrayKind(lav)) { + throw new Error(`prepareAppend only works/makes sense for arrays`) + } + const value = lav?.lav.value ?? [] + // TODO[snikitin@outlook.com] I'm sure it's possible to convince Typescript + // that below is safe, but don't know how + return { + yek, + // @ts-ignore Type '{...}' is not assignable to type 'Lav' + lav: { + lav: { + kind: yek.yek.kind, + // @ts-ignore Each member of the union type has signatures, but none of those + // signatures are compatible with each other + value: value.concat(appended_lav.lav.value), + }, + }, + } + } + private stringify(yek: Yek): string { switch (yek.yek.kind) { case 'nid': @@ -221,11 +276,10 @@ async function createNode( const yek: OriginToNidYek = { yek: { kind: 'origin->nid', key: args.origin }, } - let lav: OriginToNidLav | undefined = await store.get(yek) - lav = lav ?? { lav: { kind: 'origin->nid', value: [] } } - const nidsWithThisOrigin: Nid[] = lav.lav.value - nidsWithThisOrigin.push(node.nid) - records.push({ yek, lav }) + const lav: OriginToNidLav = { + lav: { kind: 'origin->nid', value: [node.nid] }, + } + records.push(await store.prepareAppend(yek, lav)) } if (from_nid.length > 0 || to_nid.length > 0) { @@ -316,6 +370,67 @@ async function updateNode( return { ack: true } } +async function createEdge( + store: YekLavStore, + args: CreateEdgeArgs +): Promise { + // TODO[snikitin@outlook.com] Evaluate if ownership support is needed + // and implement if yes + const owned_by = 'todo' + + const createdAt: number = unixtime.now() + const edge: TEdgeJson = { + eid: generateEid(), + from_nid: args.from, + to_nid: args.to, + crtd: createdAt, + upd: createdAt, + is_sticky: false, + owned_by, + } + + const items: YekLav[] = [] + { + const yek: NidToEdgeYek = { yek: { kind: 'nid->edge', key: args.from } } + const lav: NidToEdgeLav = { lav: { kind: 'nid->edge', value: [edge] } } + items.push(await store.prepareAppend(yek, lav)) + } + { + const reverseEdge: TEdgeJson = { + ...edge, + from_nid: args.to, + to_nid: args.from, + } + const yek: NidToEdgeYek = { yek: { kind: 'nid->edge', key: args.to } } + const lav: NidToEdgeLav = { + lav: { kind: 'nid->edge', value: [reverseEdge] }, + } + items.push(await store.prepareAppend(yek, lav)) + } + await store.set(items) + return EdgeUtil.fromJson(edge) +} + +async function getNodeAllEdges( + store: YekLavStore, + nid: string +): Promise { + const yek: NidToEdgeYek = { yek: { kind: 'nid->edge', key: nid } } + const lav: NidToEdgeLav | undefined = await store.get(yek) + const ret: NodeEdges = { from_edges: [], to_edges: [] } + if (lav == null) { + return ret + } + for (const edge of lav.lav.value) { + if (edge.to_nid === nid) { + ret.from_edges.push(EdgeUtil.fromJson(edge)) + } else { + ret.to_edges.push(EdgeUtil.fromJson(edge)) + } + } + return ret +} + async function addExternalUserActivity( store: YekLavStore, origin: OriginId, @@ -461,10 +576,10 @@ export function makeLocalStorageApi( }, }, edge: { - // create: createEdge, - // get: getNodeAllEdges, - // sticky: switchEdgeStickiness, - // delete: deleteEdge, + create: (args: CreateEdgeArgs) => createEdge(store, args), + get: (nid: string, _signal?: AbortSignal) => getNodeAllEdges(store, nid), + sticky: throwUnimplementedError('edge.sticky'), + delete: throwUnimplementedError('edge.delete'), }, activity: { external: { @@ -477,8 +592,21 @@ export function makeLocalStorageApi( getExternalUserActivity(store, origin), }, association: { - // record: recordExternalAssociation, - // get: getExternalAssociation, + // TODO[snikitin@outlook.com] Replace stubs with real implementation + record: ( + _origin: { + from: OriginId + to: OriginId + }, + _body: AddUserExternalAssociationRequest, + _signal?: AbortSignal + ) => Promise.resolve({ ack: true }), + get: ( + {}: { + origin: OriginId + }, + _signal?: AbortSignal + ) => Promise.resolve({ from: [], to: [] }), }, }, external: { diff --git a/smuggler-api/src/storage_api.ts b/smuggler-api/src/storage_api.ts index 2b91660c..50c8c67a 100644 --- a/smuggler-api/src/storage_api.ts +++ b/smuggler-api/src/storage_api.ts @@ -59,8 +59,8 @@ export type BlobUploadRequestArgs = { } export type CreateEdgeArgs = { - from?: Nid - to?: Nid + from: Nid + to: Nid signal: AbortSignal } diff --git a/smuggler-api/src/typesutil.ts b/smuggler-api/src/typesutil.ts index 382d4f4b..4fb019c3 100644 --- a/smuggler-api/src/typesutil.ts +++ b/smuggler-api/src/typesutil.ts @@ -5,6 +5,7 @@ import { TNode, TNodeJson, TEdge, + TEdgeJson, } from './types' import { AccountInterface } from './auth/account_interface' @@ -92,4 +93,11 @@ export namespace EdgeUtil { account?.isAuthenticated() && account?.getUid() === edge.owned_by return res || false } + export function fromJson(edge: TEdgeJson): TEdge { + return { + ...edge, + crtd: moment.unix(edge.crtd), + upd: moment.unix(edge.upd), + } + } } From f7daecabe92e9205df52a58a0334f8ec76be8e37 Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Wed, 11 Jan 2023 08:50:34 +0000 Subject: [PATCH 08/21] TNodeSliceIterator -> INoteIterator --- elementary/src/grid/SearchGrid.tsx | 6 +++--- smuggler-api/src/node_slice_iterator.ts | 1 + smuggler-api/src/storage_api.ts | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/elementary/src/grid/SearchGrid.tsx b/elementary/src/grid/SearchGrid.tsx index 9d67d2d9..f0539bfa 100644 --- a/elementary/src/grid/SearchGrid.tsx +++ b/elementary/src/grid/SearchGrid.tsx @@ -13,7 +13,7 @@ import { SmallCard } from '../SmallCard' import { ShrinkCard } from '../ShrinkCard' import { NodeTimeBadge } from '../NodeTimeBadge' -import { TNodeSliceIterator } from 'smuggler-api' +import { INodeIterator } from 'smuggler-api' import type { TNode, StorageApi } from 'smuggler-api' import { log, isAbortError, errorise } from 'armoury' @@ -71,7 +71,7 @@ export const SearchGrid = ({ storage: StorageApi }>) => { const [search, setUpSearch] = useState<{ - iter: TNodeSliceIterator + iter: INodeIterator beagle: Beagle } | null>(null) useEffect(() => { @@ -111,7 +111,7 @@ const SearchGridScroll = ({ storage, }: React.PropsWithChildren<{ beagle: Beagle - iter: TNodeSliceIterator + iter: INodeIterator onCardClick?: (arg0: TNode) => void portable?: boolean className?: string diff --git a/smuggler-api/src/node_slice_iterator.ts b/smuggler-api/src/node_slice_iterator.ts index 98abd5a7..fa5589c9 100644 --- a/smuggler-api/src/node_slice_iterator.ts +++ b/smuggler-api/src/node_slice_iterator.ts @@ -5,6 +5,7 @@ export interface INodeIterator { next: () => Promise> total: () => number exhausted: () => boolean + abort(): void } export type GetNodesSliceFn = ({ diff --git a/smuggler-api/src/storage_api.ts b/smuggler-api/src/storage_api.ts index 50c8c67a..805f774e 100644 --- a/smuggler-api/src/storage_api.ts +++ b/smuggler-api/src/storage_api.ts @@ -1,6 +1,6 @@ import { MimeType } from 'armoury' import type { Optional } from 'armoury' -import { TNodeSliceIterator } from './node_slice_iterator' +import { INodeIterator } from './node_slice_iterator' import { TNode, NodePatchRequest, @@ -82,7 +82,7 @@ export type StorageApi = { args: CreateNodeArgs, signal?: AbortSignal ) => Promise - slice: (args: GetNodeSliceArgs) => TNodeSliceIterator + slice: (args: GetNodeSliceArgs) => INodeIterator delete: ({ nid, signal, From b21b053b20066d0733279bb1b94236160fe54b60 Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Wed, 11 Jan 2023 08:56:45 +0000 Subject: [PATCH 09/21] rm unused features of StorageApi.node.slice --- smuggler-api/src/api_datacenter.ts | 6 ++---- smuggler-api/src/steroid/node.ts | 8 ++++---- smuggler-api/src/storage_api.ts | 2 -- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/smuggler-api/src/api_datacenter.ts b/smuggler-api/src/api_datacenter.ts index 8ab6c3b3..b22724a8 100644 --- a/smuggler-api/src/api_datacenter.ts +++ b/smuggler-api/src/api_datacenter.ts @@ -348,18 +348,16 @@ const getNodesSlice: GetNodesSliceFn = async ({ } function _getNodesSliceIter({ - end_time, start_time, - limit, origin, bucket_time_size, }: GetNodeSliceArgs) { return new TNodeSliceIterator( getNodesSlice, start_time, - end_time, + undefined, bucket_time_size, - limit, + undefined, origin ) } diff --git a/smuggler-api/src/steroid/node.ts b/smuggler-api/src/steroid/node.ts index 6881489a..8e3c180c 100644 --- a/smuggler-api/src/steroid/node.ts +++ b/smuggler-api/src/steroid/node.ts @@ -21,7 +21,7 @@ import { genOriginId, } from 'armoury' import type { Optional } from 'armoury' -import { CreateNodeArgs, StorageApi } from '../storage_api' +import { CreateNodeArgs, GetNodeSliceArgs, StorageApi } from '../storage_api' import { NodeUtil } from '../typesutil' import lodash from 'lodash' @@ -314,7 +314,7 @@ export async function lookupNodes( return storage.node.get({ nid: key.nid, signal }) } else if ('webBookmark' in key) { const { id, stableUrl } = genOriginId(key.webBookmark.url) - const query = { ...SLICE_ALL, origin: { id } } + const query: GetNodeSliceArgs = { ...SLICE_ALL, origin: { id } } const iter = storage.node.slice(query) for (let node = await iter.next(); node != null; node = await iter.next()) { @@ -326,7 +326,7 @@ export async function lookupNodes( return undefined } else if ('webQuote' in key) { const { id, stableUrl } = genOriginId(key.webQuote.url) - const query = { ...SLICE_ALL, origin: { id } } + const query: GetNodeSliceArgs = { ...SLICE_ALL, origin: { id } } const iter = storage.node.slice(query) const nodes: TNode[] = [] @@ -342,7 +342,7 @@ export async function lookupNodes( return nodes } else if ('url' in key) { const { id, stableUrl } = genOriginId(key.url) - const query = { ...SLICE_ALL, origin: { id } } + const query: GetNodeSliceArgs = { ...SLICE_ALL, origin: { id } } const iter = storage.node.slice(query) const nodes: TNode[] = [] diff --git a/smuggler-api/src/storage_api.ts b/smuggler-api/src/storage_api.ts index 2b91660c..9b633eb4 100644 --- a/smuggler-api/src/storage_api.ts +++ b/smuggler-api/src/storage_api.ts @@ -40,9 +40,7 @@ export type CreateNodeArgs = { } export type GetNodeSliceArgs = { - end_time?: number start_time?: number - limit?: number origin?: OriginId bucket_time_size?: number } From 6a798b1477c9c1b00d103b1bc24a01c5d727890b Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Wed, 11 Jan 2023 22:15:10 +0000 Subject: [PATCH 10/21] node.slice: rm timebased inputs, split origin lookup --- .../src/background/suggestAssociations.ts | 2 +- archaeologist/src/omnibox/omnibox.ts | 2 +- elementary/src/grid/SearchGrid.tsx | 8 ++--- smuggler-api/src/api_datacenter.ts | 29 +++++++++++++++++-- smuggler-api/src/node_slice_iterator.ts | 1 + smuggler-api/src/steroid/node.ts | 27 ++++++++--------- smuggler-api/src/storage_api.ts | 17 ++++++----- smuggler-api/src/storage_api_throwing.ts | 1 + 8 files changed, 56 insertions(+), 31 deletions(-) diff --git a/archaeologist/src/background/suggestAssociations.ts b/archaeologist/src/background/suggestAssociations.ts index 2519c0c7..60996dc5 100644 --- a/archaeologist/src/background/suggestAssociations.ts +++ b/archaeologist/src/background/suggestAssociations.ts @@ -7,7 +7,7 @@ export async function suggestAssociations( limit?: number ): Promise { const beagle = Beagle.fromString(phrase) - const iter = storage.node.slice({}) + const iter = storage.node.slice() const suggested: TNode[] = [] limit = limit ?? 8 // FIXME(akindyakov): This is a dirty hack to limit time of search by limiting diff --git a/archaeologist/src/omnibox/omnibox.ts b/archaeologist/src/omnibox/omnibox.ts index 34ebfd92..f24f36be 100644 --- a/archaeologist/src/omnibox/omnibox.ts +++ b/archaeologist/src/omnibox/omnibox.ts @@ -52,7 +52,7 @@ const lookUpAndSuggestFor = lodash.debounce( suggest: (suggestResults: browser.Omnibox.SuggestResult[]) => void ): Promise => { const beagle = Beagle.fromString(text) - const iter = storage.node.slice({}) + const iter = storage.node.slice() const suggestions: browser.Omnibox.SuggestResult[] = [] for ( let node = await iter.next(); diff --git a/elementary/src/grid/SearchGrid.tsx b/elementary/src/grid/SearchGrid.tsx index 9d67d2d9..6c5d32f7 100644 --- a/elementary/src/grid/SearchGrid.tsx +++ b/elementary/src/grid/SearchGrid.tsx @@ -13,7 +13,7 @@ import { SmallCard } from '../SmallCard' import { ShrinkCard } from '../ShrinkCard' import { NodeTimeBadge } from '../NodeTimeBadge' -import { TNodeSliceIterator } from 'smuggler-api' +import { INodeIterator } from 'smuggler-api' import type { TNode, StorageApi } from 'smuggler-api' import { log, isAbortError, errorise } from 'armoury' @@ -71,12 +71,12 @@ export const SearchGrid = ({ storage: StorageApi }>) => { const [search, setUpSearch] = useState<{ - iter: TNodeSliceIterator + iter: INodeIterator beagle: Beagle } | null>(null) useEffect(() => { setUpSearch({ - iter: storage.node.slice({}), + iter: storage.node.slice(), beagle: Beagle.fromString(q || undefined), }) }, [q]) @@ -111,7 +111,7 @@ const SearchGridScroll = ({ storage, }: React.PropsWithChildren<{ beagle: Beagle - iter: TNodeSliceIterator + iter: INodeIterator onCardClick?: (arg0: TNode) => void portable?: boolean className?: string diff --git a/smuggler-api/src/api_datacenter.ts b/smuggler-api/src/api_datacenter.ts index b22724a8..bf75e7f8 100644 --- a/smuggler-api/src/api_datacenter.ts +++ b/smuggler-api/src/api_datacenter.ts @@ -32,7 +32,6 @@ import { } from './types' import type { CreateNodeArgs, - GetNodeSliceArgs, NodeBatchRequestBody, CreateEdgeArgs, StorageApi, @@ -227,6 +226,25 @@ async function getNode({ return node } +async function getNodesByOrigin({ + origin, +}: { + origin: OriginId + signal?: AbortSignal +}): Promise { + const sliceAll: GetNodeSliceArgs = { + start_time: 0, // since the beginning of time + bucket_time_size: 366 * 24 * 60 * 60, + origin, + } + const ret: TNode[] = [] + const iter = _getNodesSliceIter(sliceAll) + for (let node = await iter.next(); node != null; node = await iter.next()) { + ret.push(node) + } + return ret +} + async function getNodeBatch( req: NodeBatchRequestBody, signal?: AbortSignal @@ -347,6 +365,12 @@ const getNodesSlice: GetNodesSliceFn = async ({ } } +type GetNodeSliceArgs = { + start_time?: number + origin?: OriginId + bucket_time_size?: number +} + function _getNodesSliceIter({ start_time, origin, @@ -780,9 +804,10 @@ export function makeDatacenterStorageApi(): StorageApi { return { node: { get: getNode, + getByOrigin: getNodesByOrigin, update: updateNode, create: createNode, - slice: _getNodesSliceIter, + slice: () => _getNodesSliceIter({}), delete: deleteNode, bulkDelete: bulkDeleteNodes, batch: { diff --git a/smuggler-api/src/node_slice_iterator.ts b/smuggler-api/src/node_slice_iterator.ts index 98abd5a7..5e364dd0 100644 --- a/smuggler-api/src/node_slice_iterator.ts +++ b/smuggler-api/src/node_slice_iterator.ts @@ -5,6 +5,7 @@ export interface INodeIterator { next: () => Promise> total: () => number exhausted: () => boolean + abort: () => void } export type GetNodesSliceFn = ({ diff --git a/smuggler-api/src/steroid/node.ts b/smuggler-api/src/steroid/node.ts index 8e3c180c..1a74a42c 100644 --- a/smuggler-api/src/steroid/node.ts +++ b/smuggler-api/src/steroid/node.ts @@ -21,7 +21,7 @@ import { genOriginId, } from 'armoury' import type { Optional } from 'armoury' -import { CreateNodeArgs, GetNodeSliceArgs, StorageApi } from '../storage_api' +import { CreateNodeArgs, StorageApi } from '../storage_api' import { NodeUtil } from '../typesutil' import lodash from 'lodash' @@ -314,10 +314,9 @@ export async function lookupNodes( return storage.node.get({ nid: key.nid, signal }) } else if ('webBookmark' in key) { const { id, stableUrl } = genOriginId(key.webBookmark.url) - const query: GetNodeSliceArgs = { ...SLICE_ALL, origin: { id } } - const iter = storage.node.slice(query) + const nodes: TNode[] = await storage.node.getByOrigin({ origin: { id } }) - for (let node = await iter.next(); node != null; node = await iter.next()) { + for (const node of nodes) { const nodeUrl = node.extattrs?.web?.url if (nodeUrl && stabiliseUrlForOriginId(nodeUrl) === stableUrl) { return node @@ -326,34 +325,32 @@ export async function lookupNodes( return undefined } else if ('webQuote' in key) { const { id, stableUrl } = genOriginId(key.webQuote.url) - const query: GetNodeSliceArgs = { ...SLICE_ALL, origin: { id } } - const iter = storage.node.slice(query) + const nodes: TNode[] = await storage.node.getByOrigin({ origin: { id } }) - const nodes: TNode[] = [] - for (let node = await iter.next(); node != null; node = await iter.next()) { + const ret: TNode[] = [] + for (const node of nodes) { if (NodeUtil.isWebQuote(node) && node.extattrs?.web_quote) { if ( stabiliseUrlForOriginId(node.extattrs.web_quote.url) === stableUrl ) { - nodes.push(node) + ret.push(node) } } } return nodes } else if ('url' in key) { const { id, stableUrl } = genOriginId(key.url) - const query: GetNodeSliceArgs = { ...SLICE_ALL, origin: { id } } - const iter = storage.node.slice(query) + const nodes: TNode[] = await storage.node.getByOrigin({ origin: { id } }) - const nodes: TNode[] = [] - for (let node = await iter.next(); node != null; node = await iter.next()) { + const ret: TNode[] = [] + for (const node of nodes) { if (NodeUtil.isWebBookmark(node) && node.extattrs?.web) { if (stabiliseUrlForOriginId(node.extattrs.web.url) === stableUrl) { - nodes.push(node) + ret.push(node) } } } - return nodes + return ret } throw new Error(`Failed to lookup nodes, unsupported key ${key}`) diff --git a/smuggler-api/src/storage_api.ts b/smuggler-api/src/storage_api.ts index 9b633eb4..c3ae24bc 100644 --- a/smuggler-api/src/storage_api.ts +++ b/smuggler-api/src/storage_api.ts @@ -1,6 +1,6 @@ import { MimeType } from 'armoury' import type { Optional } from 'armoury' -import { TNodeSliceIterator } from './node_slice_iterator' +import { INodeIterator } from './node_slice_iterator' import { TNode, NodePatchRequest, @@ -39,12 +39,6 @@ export type CreateNodeArgs = { created_at?: Date } -export type GetNodeSliceArgs = { - start_time?: number - origin?: OriginId - bucket_time_size?: number -} - export type NodeBatchRequestBody = { nids: Nid[] } @@ -72,6 +66,13 @@ export type SwitchEdgeStickinessArgs = { export type StorageApi = { node: { get: ({ nid, signal }: { nid: Nid; signal?: AbortSignal }) => Promise + getByOrigin: ({ + origin, + signal, + }: { + origin: OriginId + signal?: AbortSignal + }) => Promise update: ( args: { nid: Nid } & NodePatchRequest, signal?: AbortSignal @@ -80,7 +81,7 @@ export type StorageApi = { args: CreateNodeArgs, signal?: AbortSignal ) => Promise - slice: (args: GetNodeSliceArgs) => TNodeSliceIterator + slice: () => INodeIterator delete: ({ nid, signal, diff --git a/smuggler-api/src/storage_api_throwing.ts b/smuggler-api/src/storage_api_throwing.ts index 391e0cc9..e6296739 100644 --- a/smuggler-api/src/storage_api_throwing.ts +++ b/smuggler-api/src/storage_api_throwing.ts @@ -14,6 +14,7 @@ export function makeAlwaysThrowingStorageApi(): StorageApi { return { node: { get: throwError, + getByOrigin: throwError, update: throwError, create: throwError, slice: throwError, From 5d94d7c7e53c2954d9eb79f7126fbb4def1b4aa5 Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Wed, 11 Jan 2023 22:19:53 +0000 Subject: [PATCH 11/21] slice -> iterate --- archaeologist/src/background/suggestAssociations.ts | 2 +- archaeologist/src/omnibox/omnibox.ts | 2 +- elementary/src/grid/SearchGrid.tsx | 2 +- smuggler-api/src/api_datacenter.ts | 2 +- smuggler-api/src/storage_api.ts | 2 +- smuggler-api/src/storage_api_throwing.ts | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/archaeologist/src/background/suggestAssociations.ts b/archaeologist/src/background/suggestAssociations.ts index 60996dc5..8c9682ab 100644 --- a/archaeologist/src/background/suggestAssociations.ts +++ b/archaeologist/src/background/suggestAssociations.ts @@ -7,7 +7,7 @@ export async function suggestAssociations( limit?: number ): Promise { const beagle = Beagle.fromString(phrase) - const iter = storage.node.slice() + const iter = storage.node.iterate() const suggested: TNode[] = [] limit = limit ?? 8 // FIXME(akindyakov): This is a dirty hack to limit time of search by limiting diff --git a/archaeologist/src/omnibox/omnibox.ts b/archaeologist/src/omnibox/omnibox.ts index f24f36be..c072da97 100644 --- a/archaeologist/src/omnibox/omnibox.ts +++ b/archaeologist/src/omnibox/omnibox.ts @@ -52,7 +52,7 @@ const lookUpAndSuggestFor = lodash.debounce( suggest: (suggestResults: browser.Omnibox.SuggestResult[]) => void ): Promise => { const beagle = Beagle.fromString(text) - const iter = storage.node.slice() + const iter = storage.node.iterate() const suggestions: browser.Omnibox.SuggestResult[] = [] for ( let node = await iter.next(); diff --git a/elementary/src/grid/SearchGrid.tsx b/elementary/src/grid/SearchGrid.tsx index 6c5d32f7..5e4a9bca 100644 --- a/elementary/src/grid/SearchGrid.tsx +++ b/elementary/src/grid/SearchGrid.tsx @@ -76,7 +76,7 @@ export const SearchGrid = ({ } | null>(null) useEffect(() => { setUpSearch({ - iter: storage.node.slice(), + iter: storage.node.iterate(), beagle: Beagle.fromString(q || undefined), }) }, [q]) diff --git a/smuggler-api/src/api_datacenter.ts b/smuggler-api/src/api_datacenter.ts index bf75e7f8..1e2a6d54 100644 --- a/smuggler-api/src/api_datacenter.ts +++ b/smuggler-api/src/api_datacenter.ts @@ -807,7 +807,7 @@ export function makeDatacenterStorageApi(): StorageApi { getByOrigin: getNodesByOrigin, update: updateNode, create: createNode, - slice: () => _getNodesSliceIter({}), + iterate: () => _getNodesSliceIter({}), delete: deleteNode, bulkDelete: bulkDeleteNodes, batch: { diff --git a/smuggler-api/src/storage_api.ts b/smuggler-api/src/storage_api.ts index c3ae24bc..0004c562 100644 --- a/smuggler-api/src/storage_api.ts +++ b/smuggler-api/src/storage_api.ts @@ -81,7 +81,7 @@ export type StorageApi = { args: CreateNodeArgs, signal?: AbortSignal ) => Promise - slice: () => INodeIterator + iterate: () => INodeIterator delete: ({ nid, signal, diff --git a/smuggler-api/src/storage_api_throwing.ts b/smuggler-api/src/storage_api_throwing.ts index e6296739..f06920af 100644 --- a/smuggler-api/src/storage_api_throwing.ts +++ b/smuggler-api/src/storage_api_throwing.ts @@ -17,7 +17,7 @@ export function makeAlwaysThrowingStorageApi(): StorageApi { getByOrigin: throwError, update: throwError, create: throwError, - slice: throwError, + iterate: throwError, delete: throwError, bulkDelete: throwError, batch: { From c407211cfe0ac0cf187312925b79285eb35e9c66 Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Wed, 11 Jan 2023 22:36:13 +0000 Subject: [PATCH 12/21] stub new search methods --- archaeologist/src/storage_api_local.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/archaeologist/src/storage_api_local.ts b/archaeologist/src/storage_api_local.ts index bbf50301..4b38d88e 100644 --- a/archaeologist/src/storage_api_local.ts +++ b/archaeologist/src/storage_api_local.ts @@ -16,7 +16,6 @@ import { CreateNodeArgs, EdgeUtil, Eid, - GetNodeSliceArgs, NewNodeResponse, Nid, NodeBatch, @@ -31,7 +30,7 @@ import { TEdgeJson, TNode, TNodeJson, - TNodeSliceIterator, + INodeIterator, TotalUserActivity, UserExternalPipelineId, UserExternalPipelineIngestionProgress, @@ -540,17 +539,14 @@ export function makeLocalStorageApi( node: { get: ({ nid }: { nid: string; signal?: AbortSignal }) => getNode({ store, nid }), + getByOrigin: throwUnimplementedError('node.getByOrigin'), update: ( args: { nid: string } & NodePatchRequest, _signal?: AbortSignal ) => updateNode(store, args), create: (args: CreateNodeArgs, _signal?: AbortSignal) => createNode(store, args), - // TODO[snikitin@outlook.com] Local-hosted slicing implementation is a - // problem because the datacenter-hosted version depends entirely on - // time range search which is easy with in SQL, but with a KV-store - // requires to load all nodes from memory on every "iteration" - slice: throwUnimplementedError('node.slice'), + iterate: throwUnimplementedError('node.iterate'), delete: throwUnimplementedError('node.delete'), bulkDelete: throwUnimplementedError('node.bulkdDelete'), batch: { From cd5c7048972a5d28a3e6c650951d09b1382ef16e Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Wed, 11 Jan 2023 22:43:05 +0000 Subject: [PATCH 13/21] implement node.getByOrigin --- archaeologist/src/storage_api_local.ts | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/archaeologist/src/storage_api_local.ts b/archaeologist/src/storage_api_local.ts index 4b38d88e..ff681e01 100644 --- a/archaeologist/src/storage_api_local.ts +++ b/archaeologist/src/storage_api_local.ts @@ -40,7 +40,7 @@ import { v4 as uuidv4 } from 'uuid' import base32Encode from 'base32-encode' import browser from 'webextension-polyfill' -import { MimeType, unixtime } from 'armoury' +import { genOriginId, MimeType, unixtime } from 'armoury' import lodash from 'lodash' // TODO[snikitin@outlook.com] Describe that "yek" is "key" in reverse, @@ -338,6 +338,26 @@ async function getNode({ return NodeUtil.fromJson(value) } +async function getNodesByOrigin({ + store, + origin, +}: { + store: YekLavStore + origin: OriginId +}): Promise { + const yek: OriginToNidYek = { yek: { kind: 'origin->nid', key: origin } } + const lav: OriginToNidLav | undefined = await store.get(yek) + if (lav == null) { + return [] + } + const value: Nid[] = lav.lav.value + const nidYeks: NidYek[] = value.map((nid: Nid): NidYek => { + return { yek: { kind: 'nid', key: nid } } + }) + const nidLavs: NidLav[] = await store.get(nidYeks) + return nidLavs.map((lav: NidLav) => NodeUtil.fromJson(lav.lav.value)) +} + async function getNodeBatch( store: YekLavStore, req: NodeBatchRequestBody @@ -539,7 +559,8 @@ export function makeLocalStorageApi( node: { get: ({ nid }: { nid: string; signal?: AbortSignal }) => getNode({ store, nid }), - getByOrigin: throwUnimplementedError('node.getByOrigin'), + getByOrigin: ({ origin }: { origin: OriginId; signal?: AbortSignal }) => + getNodesByOrigin({ store, origin }), update: ( args: { nid: string } & NodePatchRequest, _signal?: AbortSignal From fd6f1788b80f2729d937e7f270e75f0e28ecf13e Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Thu, 12 Jan 2023 08:36:53 +0000 Subject: [PATCH 14/21] implement iterate --- archaeologist/src/storage_api_local.ts | 125 ++++++++++++++++++------ elementary/src/grid/SearchGrid.tsx | 4 +- smuggler-api/src/node_slice_iterator.ts | 1 - 3 files changed, 96 insertions(+), 34 deletions(-) diff --git a/archaeologist/src/storage_api_local.ts b/archaeologist/src/storage_api_local.ts index ff681e01..89577ed5 100644 --- a/archaeologist/src/storage_api_local.ts +++ b/archaeologist/src/storage_api_local.ts @@ -62,8 +62,11 @@ type GenericLav = { } } -type NidYek = GenericYek<'nid', Nid> -type NidLav = GenericLav<'nid', TNodeJson> +type AllNidsYek = GenericYek<'all-nids', undefined> +type AllNidsLav = GenericLav<'all-nids', Nid[]> + +type NidToNodeYek = GenericYek<'nid->node', Nid> +type NidToNodeLav = GenericLav<'nid->node', TNodeJson> type OriginToNidYek = GenericYek<'origin->nid', OriginId> type OriginToNidLav = GenericLav<'origin->nid', Nid[]> @@ -74,20 +77,22 @@ type NidToEdgeLav = GenericLav<'nid->edge', TEdgeJson[]> type OriginToActivityYek = GenericYek<'origin->activity', OriginId> type OriginToActivityLav = GenericLav<'origin->activity', TotalUserActivity> -type ExtPipelineYek = GenericYek<'ext-pipe', UserExternalPipelineId> +type ExtPipelineYek = GenericYek<'ext-pipe->progress', UserExternalPipelineId> type ExtPipelineLav = GenericLav< - 'ext-pipe', + 'ext-pipe->progress', Omit > type Yek = - | NidYek + | AllNidsYek + | NidToNodeYek | OriginToNidYek | NidToEdgeYek | OriginToActivityYek | ExtPipelineYek type Lav = - | NidLav + | AllNidsLav + | NidToNodeLav | OriginToNidLav | NidToEdgeLav | OriginToActivityLav @@ -95,7 +100,9 @@ type Lav = type YekLav = { yek: Yek; lav: Lav } -function isOfArrayKind(lav: Lav): lav is OriginToNidLav | NidToEdgeLav { +function isOfArrayKind( + lav: Lav +): lav is OriginToNidLav | NidToEdgeLav | AllNidsLav { return Array.isArray(lav.lav.value) } @@ -129,10 +136,11 @@ class YekLavStore { return this.store.set(records) } - get(yek: NidYek): Promise + get(yek: AllNidsYek): Promise + get(yek: NidToNodeYek): Promise get(yek: OriginToNidYek): Promise get(yek: NidToEdgeYek): Promise - get(yek: NidYek[]): Promise + get(yek: NidToNodeYek[]): Promise get(yek: OriginToActivityYek): Promise get(yek: ExtPipelineYek): Promise get(yek: Yek): Promise @@ -163,6 +171,13 @@ class YekLavStore { // TODO[snikitin@outlook.com] Explain that this method is a poor man's attempt // to increase atomicity of data insertion + async prepareAppend( + yek: AllNidsYek, + lav: AllNidsLav + ): Promise<{ + yek: AllNidsYek + lav: AllNidsLav + }> async prepareAppend( yek: OriginToNidYek, lav: OriginToNidLav @@ -206,15 +221,17 @@ class YekLavStore { private stringify(yek: Yek): string { switch (yek.yek.kind) { - case 'nid': - return 'nid:' + yek.yek.key + case 'all-nids': + return 'all-nids' + case 'nid->node': + return 'nid->node:' + yek.yek.key case 'origin->nid': return 'origin->nid:' + yek.yek.key.id case 'nid->edge': return 'nid->edge:' + yek.yek.key case 'origin->activity': return 'origin->activity:' + yek.yek.key.id - case 'ext-pipe': + case 'ext-pipe->progress': return 'ext-pipe:' + yek.yek.key.pipeline_key } } @@ -264,10 +281,14 @@ async function createNode( updated_at: createdAt, } - let records: { yek: Yek; lav: Lav }[] = [ + let records: YekLav[] = [ + await store.prepareAppend( + { yek: { kind: 'all-nids', key: undefined } }, + { lav: { kind: 'all-nids', value: [node.nid] } } + ), { - yek: { yek: { kind: 'nid', key: node.nid } }, - lav: { lav: { kind: 'nid', value: node } }, + yek: { yek: { kind: 'nid->node', key: node.nid } }, + lav: { lav: { kind: 'nid->node', value: node } }, }, ] @@ -329,8 +350,8 @@ async function getNode({ store: YekLavStore nid: Nid }): Promise { - const yek: NidYek = { yek: { kind: 'nid', key: nid } } - const lav: NidLav | undefined = await store.get(yek) + const yek: NidToNodeYek = { yek: { kind: 'nid->node', key: nid } } + const lav: NidToNodeLav | undefined = await store.get(yek) if (lav == null) { throw new Error(`Failed to get node ${nid} because it wasn't found`) } @@ -351,30 +372,32 @@ async function getNodesByOrigin({ return [] } const value: Nid[] = lav.lav.value - const nidYeks: NidYek[] = value.map((nid: Nid): NidYek => { - return { yek: { kind: 'nid', key: nid } } + const nidYeks: NidToNodeYek[] = value.map((nid: Nid): NidToNodeYek => { + return { yek: { kind: 'nid->node', key: nid } } }) - const nidLavs: NidLav[] = await store.get(nidYeks) - return nidLavs.map((lav: NidLav) => NodeUtil.fromJson(lav.lav.value)) + const nidLavs: NidToNodeLav[] = await store.get(nidYeks) + return nidLavs.map((lav: NidToNodeLav) => NodeUtil.fromJson(lav.lav.value)) } async function getNodeBatch( store: YekLavStore, req: NodeBatchRequestBody ): Promise { - const yeks: NidYek[] = req.nids.map((nid: Nid): NidYek => { - return { yek: { kind: 'nid', key: nid } } + const yeks: NidToNodeYek[] = req.nids.map((nid: Nid): NidToNodeYek => { + return { yek: { kind: 'nid->node', key: nid } } }) - const lavs: NidLav[] = await store.get(yeks) - return { nodes: lavs.map((lav: NidLav) => NodeUtil.fromJson(lav.lav.value)) } + const lavs: NidToNodeLav[] = await store.get(yeks) + return { + nodes: lavs.map((lav: NidToNodeLav) => NodeUtil.fromJson(lav.lav.value)), + } } async function updateNode( store: YekLavStore, args: { nid: Nid } & NodePatchRequest ): Promise { - const yek: NidYek = { yek: { kind: 'nid', key: args.nid } } - const lav: NidLav | undefined = await store.get(yek) + const yek: NidToNodeYek = { yek: { kind: 'nid->node', key: args.nid } } + const lav: NidToNodeLav | undefined = await store.get(yek) if (lav == null) { throw new Error(`Failed to update node ${args.nid} because it wasn't found`) } @@ -389,6 +412,46 @@ async function updateNode( return { ack: true } } +class Iterator implements INodeIterator { + private store: YekLavStore + private nids: Promise + private index: number + + constructor(store: YekLavStore) { + this.store = store + this.index = 0 + + const yek: AllNidsYek = { + yek: { kind: 'all-nids', key: undefined }, + } + this.nids = store + .get(yek) + .then((lav: AllNidsLav | undefined) => lav?.lav.value ?? []) + } + + async next(): Promise { + const nids = await this.nids + if (this.index >= nids.length) { + return null + } + const nid: Nid = nids[this.index] + const yek: NidToNodeYek = { yek: { kind: 'nid->node', key: nid } } + const lav: NidToNodeLav | undefined = await this.store.get(yek) + if (lav == null) { + throw new Error(`Failed to find node for nid ${nid}`) + } + ++this.index + return NodeUtil.fromJson(lav.lav.value) + } + total(): number { + return this.index + } + abort(): void { + this.index = 0 + this.nids = Promise.resolve([]) + } +} + async function createEdge( store: YekLavStore, args: CreateEdgeArgs @@ -507,7 +570,7 @@ async function getUserIngestionProgress( epid: UserExternalPipelineId ): Promise { const yek: ExtPipelineYek = { - yek: { kind: 'ext-pipe', key: epid }, + yek: { kind: 'ext-pipe->progress', key: epid }, } const lav: ExtPipelineLav | undefined = await store.get(yek) if (lav == null) { @@ -533,10 +596,10 @@ async function advanceUserIngestionProgress( progress.ingested_until = new_progress.ingested_until const yek: ExtPipelineYek = { - yek: { kind: 'ext-pipe', key: epid }, + yek: { kind: 'ext-pipe->progress', key: epid }, } const lav: ExtPipelineLav = { - lav: { kind: 'ext-pipe', value: progress }, + lav: { kind: 'ext-pipe->progress', value: progress }, } await store.set([{ yek, lav }]) return { ack: true } @@ -567,7 +630,7 @@ export function makeLocalStorageApi( ) => updateNode(store, args), create: (args: CreateNodeArgs, _signal?: AbortSignal) => createNode(store, args), - iterate: throwUnimplementedError('node.iterate'), + iterate: () => new Iterator(store), delete: throwUnimplementedError('node.delete'), bulkDelete: throwUnimplementedError('node.bulkdDelete'), batch: { diff --git a/elementary/src/grid/SearchGrid.tsx b/elementary/src/grid/SearchGrid.tsx index 5e4a9bca..69e9c65b 100644 --- a/elementary/src/grid/SearchGrid.tsx +++ b/elementary/src/grid/SearchGrid.tsx @@ -150,9 +150,9 @@ const SearchGridScroll = ({ setFetching(true) try { while (isScrolledToBottom()) { - const node = await iter.next() + const node = await (await iter).next() if (node == null) { - iter.abort() + ;(await iter).abort() break } if (beagle.searchNode(node) != null) { diff --git a/smuggler-api/src/node_slice_iterator.ts b/smuggler-api/src/node_slice_iterator.ts index 5e364dd0..8608e298 100644 --- a/smuggler-api/src/node_slice_iterator.ts +++ b/smuggler-api/src/node_slice_iterator.ts @@ -4,7 +4,6 @@ import type { Optional } from 'armoury' export interface INodeIterator { next: () => Promise> total: () => number - exhausted: () => boolean abort: () => void } From d201905cea948d6179f6ec3f89ec1a8527bfaf48 Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Thu, 12 Jan 2023 09:13:07 +0000 Subject: [PATCH 15/21] renames, docs --- ...pi_local.ts => storage_api_browser_ext.ts} | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) rename archaeologist/src/{storage_api_local.ts => storage_api_browser_ext.ts} (95%) diff --git a/archaeologist/src/storage_api_local.ts b/archaeologist/src/storage_api_browser_ext.ts similarity index 95% rename from archaeologist/src/storage_api_local.ts rename to archaeologist/src/storage_api_browser_ext.ts index 89577ed5..4176ea16 100644 --- a/archaeologist/src/storage_api_local.ts +++ b/archaeologist/src/storage_api_browser_ext.ts @@ -1,10 +1,16 @@ /** * An implementation of @see smuggler-api.StorageApi that persists the data - * on user device and is based on browser extension APIs. + * in a storage only available to browser extensions (most likely - locally on device). * * It is intended to work in all browser extension contexts (such as background, - * content etc). See https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/storage + * content etc). + * See https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/storage * for more information. + * + * NOTE: At the time of this writing dev tools of some browsers (including the + * Chromium-based ones) do not including anything to inspect the contents + * of the underlying storage. See https://stackoverflow.com/a/27432365/3375765 + * for workarounds. */ import { @@ -117,6 +123,10 @@ class YekLavStore { this.store = store } + // TODO[snikitin@outlook.com] Add a note that the code that uses this + // method should try to combine everything it needs to do into ONE + // call to set() in hopes to retain at least some data consistency + // guarantees. set(items: YekLav[]): Promise { for (const item of items) { if (item.yek.yek.kind !== item.lav.lav.kind) { @@ -128,7 +138,7 @@ class YekLavStore { let records: Record = {} for (const item of items) { const key = this.stringify(item.yek) - if (records.keys().indexOf(key) !== -1) { + if (Object.keys(records).indexOf(key) !== -1) { throw new Error(`Attempted to set more than 1 value for key '${key}'`) } records[key] = item.lav @@ -605,7 +615,7 @@ async function advanceUserIngestionProgress( return { ack: true } } -export function makeLocalStorageApi( +export function makeBrowserExtStorageApi( browserStore: browser.Storage.StorageArea ): StorageApi { const store = new YekLavStore(browserStore) @@ -613,7 +623,8 @@ export function makeLocalStorageApi( const throwUnimplementedError = (endpoint: string) => { return (..._: any[]): never => { throw new Error( - `Attempted to call an ${endpoint} endpoint of local StorageApi which hasn't been implemented yet` + `Attempted to call an ${endpoint} endpoint of browser ` + + "extension's local StorageApi which hasn't been implemented yet" ) } } From 974826f0c96febad2909ea136fdc53fd521a24c2 Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Thu, 12 Jan 2023 09:26:51 +0000 Subject: [PATCH 16/21] lowercase nids --- archaeologist/src/storage_api_browser_ext.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archaeologist/src/storage_api_browser_ext.ts b/archaeologist/src/storage_api_browser_ext.ts index 4176ea16..5bbcfd43 100644 --- a/archaeologist/src/storage_api_browser_ext.ts +++ b/archaeologist/src/storage_api_browser_ext.ts @@ -255,7 +255,7 @@ function generateNid(): Nid { */ const uuid = uuidv4() const array: Uint8Array = new TextEncoder().encode(uuid) - return base32Encode(array, 'Crockford') + return base32Encode(array, 'Crockford').toLowerCase() } function generateEid(): Eid { From 12b46053f3630e411cd3751f14e87f1ec9f86ce5 Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Thu, 12 Jan 2023 10:23:06 +0000 Subject: [PATCH 17/21] lint --- archaeologist/src/storage_api_browser_ext.ts | 6 +++--- truthsayer/src/card/Triptych.tsx | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/archaeologist/src/storage_api_browser_ext.ts b/archaeologist/src/storage_api_browser_ext.ts index 5bbcfd43..e7861cdb 100644 --- a/archaeologist/src/storage_api_browser_ext.ts +++ b/archaeologist/src/storage_api_browser_ext.ts @@ -46,7 +46,7 @@ import { v4 as uuidv4 } from 'uuid' import base32Encode from 'base32-encode' import browser from 'webextension-polyfill' -import { genOriginId, MimeType, unixtime } from 'armoury' +import { MimeType, unixtime } from 'armoury' import lodash from 'lodash' // TODO[snikitin@outlook.com] Describe that "yek" is "key" in reverse, @@ -272,7 +272,7 @@ async function createNode( args: CreateNodeArgs ): Promise { // TODO[snikitin@outlook.com] Below keys must become functional somehow. - const created_via: NodeCreatedVia | undefined = args.created_via + const _created_via: NodeCreatedVia | undefined = args.created_via // TODO[snikitin@outlook.com] This graph structure has to work somehow const from_nid: Nid[] = args.from_nid ?? [] @@ -693,7 +693,7 @@ export function makeBrowserExtStorageApi( _signal?: AbortSignal ) => Promise.resolve({ ack: true }), get: ( - {}: { + _args: { origin: OriginId }, _signal?: AbortSignal diff --git a/truthsayer/src/card/Triptych.tsx b/truthsayer/src/card/Triptych.tsx index 7a8af2f1..43ca76f8 100644 --- a/truthsayer/src/card/Triptych.tsx +++ b/truthsayer/src/card/Triptych.tsx @@ -301,7 +301,7 @@ export class Triptych extends React.Component { } render() { - const { node, edges_sticky, edges_right, edges_left } = this.state + const { node, edges_right, edges_left } = this.state const nodeCard = node !== null ? ( Date: Thu, 12 Jan 2023 10:25:11 +0000 Subject: [PATCH 18/21] rm old experiment --- elementary/src/grid/SearchGrid.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elementary/src/grid/SearchGrid.tsx b/elementary/src/grid/SearchGrid.tsx index 0eaf8df4..1b08524a 100644 --- a/elementary/src/grid/SearchGrid.tsx +++ b/elementary/src/grid/SearchGrid.tsx @@ -149,9 +149,9 @@ const SearchGridScroll = ({ setFetching(true) try { while (isScrolledToBottom()) { - const node = await (await iter).next() + const node = await iter.next() if (node == null) { - ;(await iter).abort() + iter.abort() break } if (beagle.searchNode(node) != null) { From c524a546d52af4df4c7a3b5425f40b277531f360 Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Thu, 12 Jan 2023 10:26:08 +0000 Subject: [PATCH 19/21] docs --- archaeologist/src/storage_api_browser_ext.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archaeologist/src/storage_api_browser_ext.ts b/archaeologist/src/storage_api_browser_ext.ts index e7861cdb..6a03e8ec 100644 --- a/archaeologist/src/storage_api_browser_ext.ts +++ b/archaeologist/src/storage_api_browser_ext.ts @@ -8,7 +8,7 @@ * for more information. * * NOTE: At the time of this writing dev tools of some browsers (including the - * Chromium-based ones) do not including anything to inspect the contents + * Chromium-based ones) do not include anything that would allow to inspect the contents * of the underlying storage. See https://stackoverflow.com/a/27432365/3375765 * for workarounds. */ From 5a5b8dcd18989652d629830f079bd9b3cd224536 Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Sat, 14 Jan 2023 16:07:52 +0000 Subject: [PATCH 20/21] lint --- archaeologist/src/storage_api_browser_ext.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archaeologist/src/storage_api_browser_ext.ts b/archaeologist/src/storage_api_browser_ext.ts index 6a03e8ec..c4355442 100644 --- a/archaeologist/src/storage_api_browser_ext.ts +++ b/archaeologist/src/storage_api_browser_ext.ts @@ -272,7 +272,7 @@ async function createNode( args: CreateNodeArgs ): Promise { // TODO[snikitin@outlook.com] Below keys must become functional somehow. - const _created_via: NodeCreatedVia | undefined = args.created_via + // const _created_via: NodeCreatedVia | undefined = args.created_via // TODO[snikitin@outlook.com] This graph structure has to work somehow const from_nid: Nid[] = args.from_nid ?? [] From 6247a2d21a70a88814e58ef090c3d9c06e4501d9 Mon Sep 17 00:00:00 2001 From: Sergey Nikitin Date: Sat, 14 Jan 2023 16:29:39 +0000 Subject: [PATCH 21/21] lint --- archaeologist/src/storage_api_browser_ext.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/archaeologist/src/storage_api_browser_ext.ts b/archaeologist/src/storage_api_browser_ext.ts index c4355442..ec2a9c49 100644 --- a/archaeologist/src/storage_api_browser_ext.ts +++ b/archaeologist/src/storage_api_browser_ext.ts @@ -13,35 +13,31 @@ * for workarounds. */ -import { +import type { Ack, AddUserActivityRequest, AddUserExternalAssociationRequest, AdvanceExternalPipelineIngestionProgress, CreateEdgeArgs, CreateNodeArgs, - EdgeUtil, Eid, NewNodeResponse, Nid, NodeBatch, NodeBatchRequestBody, - NodeCreatedVia, NodeEdges, NodePatchRequest, - NodeUtil, OriginId, StorageApi, TEdge, TEdgeJson, TNode, TNodeJson, - INodeIterator, TotalUserActivity, UserExternalPipelineId, UserExternalPipelineIngestionProgress, } from 'smuggler-api' -import { NodeType } from 'smuggler-api' +import { INodeIterator, NodeUtil, EdgeUtil, NodeType } from 'smuggler-api' import { v4 as uuidv4 } from 'uuid' import base32Encode from 'base32-encode'