Skip to content

Commit 99c3031

Browse files
OleksiiZubkoOleksiiZubko
authored
An example web app. (#460)
An example web app shows how to use Here Data SDK for TypeScript to upload and publish large data in a browser. Resolves: OLPEDGE-2501 Relates-To: OLPEDGE-2456 Signed-off-by: Oleksii Zubko <ext-oleksii.zubko@here.com>
1 parent 2de0e17 commit 99c3031

File tree

4 files changed

+324
-0
lines changed

4 files changed

+324
-0
lines changed

‎examples/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ HERE Data SDK for TypeScript contains several examples that demonstrate some of
55
- [Authorization example](authorization-example/README.md) – build the Data SDK using webpack and work with the APIs from `@here/olp-sdk-dataservice-api`. You can use this example application to find the list of groups that you have access to as well as create groups.
66
- [React App example](react-app-example/README.md) – use the Data SDK and React in a browser and learn how to work with the following modules: `olp-sdk-authentication`, `olp-sdk-dataservice-read`, and `olp-sdk-dataservice-write`.
77
- [Node.js example](nodejs-example/README.md) – work with the Data SDK in Node.js and learn how to get data from versioned layers and save it to files.
8+
- [MultiPartUploadWrapper example](multipart-upload-wrapper-example/README.md) – use the Data SDK to upload and publish a large amount of data in a browser.

‎examples/multipart-upload-wrapper-example/README.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Publish large data files in a browser
2+
3+
This example app uses an HTML and JS UI and an HTTP server with mocked routes to simulate the real `DataStore` class.
4+
You can use this app to learn how to work with the `MultipartUploadWrapper` class and publish large files of more than 8 GB to a versioned layer without loading them in memory.
5+
6+
## Setup
7+
8+
This example does not need any setup. Nevertheless, make sure you installed Node.js.
9+
10+
## Run
11+
12+
1. Start the server.
13+
```
14+
node server.js
15+
```
16+
2. In your favorite browser, open `http://localhost:8080/`.

‎examples/multipart-upload-wrapper-example/server.js

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
/*
2+
* Copyright (C) 2021 HERE Europe B.V.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* SPDX-License-Identifier: Apache-2.0
17+
* License-Filename: LICENSE
18+
*/
19+
20+
"use strict";
21+
22+
const http = require("http");
23+
const fs = require("fs");
24+
25+
const HOST = "127.0.0.1";
26+
const PORT = 8080;
27+
const RESPONSE_DELAY_MS = 300;
28+
29+
const routing = {
30+
"/": `<!DOCTYPE html>
31+
<html lang="en">
32+
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
33+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
34+
<meta name="title" content="MultiPartUpload example webapp">
35+
<meta name=”description” content="MultiPartUpload example webapp">
36+
<body>
37+
<button id="abort">Abort</button>
38+
<div id="app"></div>
39+
<script src="https://unpkg.com/@here/olp-sdk-core/bundle.umd.min.js"></script>
40+
<script src="https://unpkg.com/@here/olp-sdk-authentication/bundle.umd.min.js"></script>
41+
<script src="https://unpkg.com/@here/olp-sdk-dataservice-api/bundle.umd.min.js"></script>
42+
<script src="https://unpkg.com/@here/olp-sdk-dataservice-write/bundle.umd.min.js"></script>
43+
<script src="ui.js"></script>
44+
</body>
45+
</html>`,
46+
"/ui.js": fs.readFileSync("./ui.js").toString(),
47+
"/oauth2/token": () => ({
48+
accessToken: "mocked-access-token",
49+
tokenType: "bearer",
50+
expiresIn: 99999999,
51+
}),
52+
"/api/lookup/resources/hrn:here:data::mocked:catalog/apis": () => [
53+
{
54+
api: "blob",
55+
version: "v1",
56+
baseURL: `http://${HOST}:${PORT}/blobstore/v1`,
57+
parameters: {},
58+
},
59+
{
60+
api: "publish",
61+
version: "v2",
62+
baseURL: `http://${HOST}:${PORT}/publish/v2`,
63+
parameters: {},
64+
},
65+
],
66+
"/publish/v2/publications": () => ({
67+
catalogId: "catalog",
68+
catalogVersion: 999,
69+
details: {
70+
expires: 9999999999999,
71+
message: "",
72+
modified: 9999999999999,
73+
started: 9999999999999,
74+
state: "initialized",
75+
},
76+
id: "mocked-publication-id",
77+
layerIds: ["mocked-layer"],
78+
versionDependencies: [],
79+
}),
80+
"/blobstore/v1/layers/mocked-layer/data/mocked-datahandle/multiparts": () => ({
81+
links: {
82+
status: {
83+
href: `http://${HOST}:${PORT}/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token`,
84+
method: "GET",
85+
},
86+
delete: {
87+
href: `http://${HOST}:${PORT}/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token`,
88+
method: "DELETE",
89+
},
90+
uploadPart: {
91+
href: `http://${HOST}:${PORT}/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token/parts`,
92+
method: "POST",
93+
},
94+
complete: {
95+
href: `http://${HOST}:${PORT}/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token`,
96+
method: "PUT",
97+
},
98+
},
99+
}),
100+
"/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token/parts": (
101+
req,
102+
res
103+
) => {
104+
res.setHeader("ETag", Math.random() * 10000);
105+
res.setHeader("Access-Control-Expose-Headers", "ETag");
106+
return {
107+
status: res.statusCode,
108+
};
109+
},
110+
"/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token": () => ({
111+
ok: true,
112+
}),
113+
"/publish/v2/layers/mocked-layer/publications/mocked-publication-id/partitions": () => ({
114+
ok: true,
115+
}),
116+
"/publish/v2/publications/mocked-publication-id": () => ({
117+
ok: true,
118+
}),
119+
};
120+
121+
const types = {
122+
object: JSON.stringify,
123+
string: (s) => s,
124+
number: (val) => `${val}`,
125+
undefined: () => "Not Found",
126+
function: (fn, req, res) => JSON.stringify(fn(req, res)),
127+
};
128+
129+
const server = http.createServer((req, res) => {
130+
console.dir({ type: "Reguest", method: req.method, url: req.url });
131+
const queryIndex = req.url.indexOf("?");
132+
const adaptedUrl =
133+
queryIndex !== -1 ? req.url.substr(0, queryIndex) : req.url;
134+
135+
const data = routing[adaptedUrl];
136+
const type = typeof data;
137+
const serialiser = types[type];
138+
const result = serialiser(data, req, res);
139+
140+
res.setHeader("Access-Control-Allow-Origin", "*");
141+
res.setHeader(
142+
"Access-Control-Allow-Headers",
143+
"authorization,content-type, ETag"
144+
);
145+
res.setHeader("Access-Control-Allow-Methods", "*");
146+
147+
res.statusCode = type === "undefined" ? 404 : 200;
148+
setTimeout(() => {
149+
res.end(result);
150+
}, RESPONSE_DELAY_MS);
151+
});
152+
153+
server.listen(PORT, HOST, () => {
154+
console.log(`Server running at http://${HOST}:${PORT}`);
155+
});

‎examples/multipart-upload-wrapper-example/ui.js

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
/*
2+
* Copyright (C) 2021 HERE Europe B.V.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
* SPDX-License-Identifier: Apache-2.0
17+
* License-Filename: LICENSE
18+
*/
19+
20+
"use strict";
21+
22+
const appContainer = document.getElementById("app");
23+
const abortController = new AbortController();
24+
25+
// Create `UserAuth` and set it up to use our mocked server.
26+
const userAuth = new UserAuth({
27+
tokenRequester: requestToken,
28+
customUrl: "http://127.0.0.1:8080/oauth2/token",
29+
credentials: {
30+
accessKeyId: "mocked-access-key-id",
31+
accessKeySecret: "mocked-access-key-secret",
32+
},
33+
});
34+
35+
// Create `OlpClientSettings` and set it up to use our mocked server.
36+
const settings = new OlpClientSettings({
37+
environment: "http://127.0.0.1:8080/api/lookup",
38+
getToken: () => userAuth.getToken(),
39+
});
40+
41+
/**
42+
* ****** Example of using HERE Data SDK for TypeScript in plain JS *******
43+
*/
44+
async function upload(file) {
45+
// Set up and draw a progress bar.
46+
const progress = document.createElement("div");
47+
const progressBar = document.createElement("progress");
48+
appContainer.appendChild(progressBar);
49+
appContainer.appendChild(progress);
50+
51+
// Set up the mocked data.
52+
const catalogHrn = "hrn:here:data::mocked:catalog";
53+
const layerId = "mocked-layer";
54+
const datahandle = "mocked-datahandle";
55+
const contentType = "text/plain";
56+
57+
/**
58+
* Create the `DataStoreRequest` builder.
59+
* You need it to make requests to the Publish API.
60+
*/
61+
const publishRequestBuilder = await RequestFactory.create(
62+
"publish",
63+
"v2",
64+
settings,
65+
HRN.fromString(catalogHrn),
66+
abortController.signal
67+
);
68+
69+
// Initialize the new publication by sending a request to the Publish API.
70+
const publication = await PublishApi.initPublication(publishRequestBuilder, {
71+
body: {
72+
layerIds: [layerId],
73+
},
74+
});
75+
76+
/**
77+
* Set up callbacks to subscribe to the uploading progress and progress bar drawing.
78+
*/
79+
let totalFileSize = 0;
80+
let uploadedSize = 0;
81+
82+
const onStart = (event) => {
83+
totalFileSize = event.dataSize;
84+
progressBar.setAttribute("max", `${totalFileSize}`);
85+
progress.innerHTML = `Processing...`;
86+
};
87+
88+
const onStatus = (status) => {
89+
uploadedSize += status.chunkSize;
90+
progressBar.setAttribute("value", `${uploadedSize}`);
91+
progress.innerHTML = `Uploaded to Blob V1 ${uploadedSize} of ${totalFileSize} bytes.
92+
Chunks ${status.uploadedChunks} of ${status.totalChunks}`;
93+
};
94+
95+
// Initialize `MultiPartUploadWrapper`.
96+
const wrapper = new MultiPartUploadWrapper(
97+
{
98+
blobVersion: "v1",
99+
catalogHrn,
100+
contentType,
101+
handle: datahandle,
102+
layerId,
103+
onStart,
104+
onStatus,
105+
},
106+
settings
107+
);
108+
109+
// Upload the file.
110+
await wrapper.upload(file, abortController.signal);
111+
112+
// Upload metadata of a new partition by sending a request to the Publish API.
113+
await PublishApi.uploadPartitions(publishRequestBuilder, {
114+
layerId,
115+
publicationId: publication.id,
116+
body: {
117+
partitions: [
118+
{
119+
partition: file.name,
120+
dataHandle: datahandle,
121+
dataSize: totalFileSize,
122+
},
123+
],
124+
},
125+
});
126+
127+
// Submit the new publication by sending a request to the Publish API.
128+
await PublishApi.submitPublication(publishRequestBuilder, {
129+
publicationId: publication.id,
130+
});
131+
132+
progress.innerHTML += "\nDone!";
133+
}
134+
135+
/**
136+
* ************** UI ******************
137+
*/
138+
const abortButton = document.getElementById("abort");
139+
abortButton.onclick = () => {
140+
abortController.abort();
141+
};
142+
143+
const label = document.createElement("label");
144+
label.innerHTML = "Publish to Blob V1: ";
145+
label.setAttribute("for", "uploadToBlobV1");
146+
appContainer.appendChild(label);
147+
148+
const input = document.createElement("input");
149+
input.setAttribute("type", "file");
150+
input.setAttribute("name", "uploadToBlobV1");
151+
input.setAttribute("onchange", "upload(this.files[0])");
152+
appContainer.appendChild(input);

0 commit comments

Comments
 (0)