Skip to content

Commit 38d26a3

Browse files
hkdemanhkdeman
committed
feat(core): fix elasticsearch update and get row logic
feat(frontend): add sidebar and storage unit name changes
1 parent 392f133 commit 38d26a3

File tree

11 files changed

+137
-97
lines changed

11 files changed

+137
-97
lines changed

‎core/src/plugins/elasticsearch/db.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,22 @@ package elasticsearch
33
import (
44
"fmt"
55

6+
"github.com/clidey/whodb/core/src/common"
67
"github.com/clidey/whodb/core/src/engine"
78
"github.com/elastic/go-elasticsearch/v8"
89
)
910

1011
func DB(config *engine.PluginConfig) (*elasticsearch.Client, error) {
1112
var addresses []string
12-
if config.Credentials.Hostname == "localhost" || config.Credentials.Hostname == "host.docker.internal" {
13+
port := common.GetRecordValueOrDefault(config.Credentials.Advanced, "Port", "9200")
14+
sslMode := common.GetRecordValueOrDefault(config.Credentials.Advanced, "SSL Mode", "disable")
15+
if sslMode == "enable" {
1316
addresses = []string{
14-
fmt.Sprintf("http://%s:%d", config.Credentials.Hostname, 9200),
17+
fmt.Sprintf("https://%s:%s", config.Credentials.Hostname, port),
1518
}
1619
} else {
1720
addresses = []string{
18-
fmt.Sprintf("https://%s:%d", config.Credentials.Hostname, 443),
21+
fmt.Sprintf("http://%s:%s", config.Credentials.Hostname, port),
1922
}
2023
}
2124

‎core/src/plugins/elasticsearch/elasticsearch.go

Lines changed: 31 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -25,36 +25,11 @@ func (p *ElasticSearchPlugin) IsAvailable(config *engine.PluginConfig) bool {
2525
}
2626

2727
func (p *ElasticSearchPlugin) GetDatabases() ([]string, error) {
28-
return nil, errors.New("unsupported operation")
28+
return nil, errors.ErrUnsupported
2929
}
3030

3131
func (p *ElasticSearchPlugin) GetSchema(config *engine.PluginConfig) ([]string, error) {
32-
client, err := DB(config)
33-
if err != nil {
34-
return nil, err
35-
}
36-
37-
res, err := client.Indices.Get([]string{})
38-
if err != nil {
39-
return nil, err
40-
}
41-
defer res.Body.Close()
42-
43-
if res.IsError() {
44-
return nil, fmt.Errorf("error getting indices: %s", res.String())
45-
}
46-
47-
var indices map[string]interface{}
48-
if err := json.NewDecoder(res.Body).Decode(&indices); err != nil {
49-
return nil, err
50-
}
51-
52-
databases := make([]string, 0, len(indices))
53-
for index := range indices {
54-
databases = append(databases, index)
55-
}
56-
57-
return databases, nil
32+
return nil, errors.ErrUnsupported
5833
}
5934

6035
func (p *ElasticSearchPlugin) GetStorageUnits(config *engine.PluginConfig, database string) ([]engine.StorageUnit, error) {
@@ -63,14 +38,14 @@ func (p *ElasticSearchPlugin) GetStorageUnits(config *engine.PluginConfig, datab
6338
return nil, err
6439
}
6540

66-
res, err := client.Indices.Stats(client.Indices.Stats.WithIndex(database))
41+
res, err := client.Indices.Stats()
6742
if err != nil {
6843
return nil, err
6944
}
7045
defer res.Body.Close()
7146

7247
if res.IsError() {
73-
return nil, fmt.Errorf("error getting stats for index %s: %s", database, res.String())
48+
return nil, fmt.Errorf("error getting stats for indices: %s", res.String())
7449
}
7550

7651
var stats map[string]interface{}
@@ -79,19 +54,22 @@ func (p *ElasticSearchPlugin) GetStorageUnits(config *engine.PluginConfig, datab
7954
}
8055

8156
indicesStats := stats["indices"].(map[string]interface{})
82-
indexStats := indicesStats[database].(map[string]interface{})
83-
primaries := indexStats["primaries"].(map[string]interface{})
84-
docs := primaries["docs"].(map[string]interface{})
85-
store := primaries["store"].(map[string]interface{})
86-
87-
storageUnits := []engine.StorageUnit{
88-
{
89-
Name: database,
57+
storageUnits := make([]engine.StorageUnit, 0, len(indicesStats))
58+
59+
for indexName, indexStatsInterface := range indicesStats {
60+
indexStats := indexStatsInterface.(map[string]interface{})
61+
primaries := indexStats["primaries"].(map[string]interface{})
62+
docs := primaries["docs"].(map[string]interface{})
63+
store := primaries["store"].(map[string]interface{})
64+
65+
storageUnit := engine.StorageUnit{
66+
Name: indexName,
9067
Attributes: []engine.Record{
9168
{Key: "Storage Size", Value: fmt.Sprintf("%v", store["size_in_bytes"])},
9269
{Key: "Count", Value: fmt.Sprintf("%v", docs["count"])},
9370
},
94-
},
71+
}
72+
storageUnits = append(storageUnits, storageUnit)
9573
}
9674

9775
return storageUnits, nil
@@ -103,17 +81,20 @@ func (p *ElasticSearchPlugin) GetRows(config *engine.PluginConfig, database, col
10381
return nil, err
10482
}
10583

106-
var esFilter map[string]interface{}
84+
var elasticSearchConditions map[string]interface{}
10785
if len(filter) > 0 {
108-
if err := json.Unmarshal([]byte(filter), &esFilter); err != nil {
86+
if err := json.Unmarshal([]byte(filter), &elasticSearchConditions); err != nil {
10987
return nil, fmt.Errorf("invalid filter format: %v", err)
11088
}
11189
}
11290

11391
query := map[string]interface{}{
114-
"from": pageOffset,
115-
"size": pageSize,
116-
"query": esFilter,
92+
"from": pageOffset,
93+
"size": pageSize,
94+
}
95+
96+
for key, value := range elasticSearchConditions {
97+
query[key] = value
11798
}
11899

119100
var buf bytes.Buffer
@@ -150,8 +131,13 @@ func (p *ElasticSearchPlugin) GetRows(config *engine.PluginConfig, database, col
150131
}
151132

152133
for _, hit := range hits {
153-
doc := hit.(map[string]interface{})["_source"]
154-
jsonBytes, err := json.Marshal(doc)
134+
hitMap := hit.(map[string]interface{})
135+
source := hitMap["_source"]
136+
id := hitMap["_id"]
137+
document := map[string]interface{}{}
138+
document["_id"] = id
139+
document["source"] = source
140+
jsonBytes, err := json.Marshal(document)
155141
if err != nil {
156142
return nil, err
157143
}

‎core/src/plugins/elasticsearch/graph.go

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ func (p *ElasticSearchPlugin) GetGraph(config *engine.PluginConfig, database str
2121
if err != nil {
2222
return nil, err
2323
}
24-
res, err := client.Indices.Get([]string{database})
24+
25+
res, err := client.Indices.Stats()
2526
if err != nil {
2627
return nil, err
2728
}
@@ -31,19 +32,15 @@ func (p *ElasticSearchPlugin) GetGraph(config *engine.PluginConfig, database str
3132
return nil, fmt.Errorf("error getting indices: %s", res.String())
3233
}
3334

34-
var indices map[string]interface{}
35-
if err := json.NewDecoder(res.Body).Decode(&indices); err != nil {
35+
var stats map[string]interface{}
36+
if err := json.NewDecoder(res.Body).Decode(&stats); err != nil {
3637
return nil, err
3738
}
3839

39-
collections := make([]string, 0, len(indices))
40-
for index := range indices {
41-
collections = append(collections, index)
42-
}
40+
indicesStats := stats["indices"].(map[string]interface{})
4341

4442
relations := []tableRelation{}
45-
46-
for _, collectionName := range collections {
43+
for indexName := range indicesStats {
4744
var buf bytes.Buffer
4845
query := map[string]interface{}{
4946
"size": 1,
@@ -57,7 +54,7 @@ func (p *ElasticSearchPlugin) GetGraph(config *engine.PluginConfig, database str
5754

5855
res, err := client.Search(
5956
client.Search.WithContext(context.Background()),
60-
client.Search.WithIndex(collectionName),
57+
client.Search.WithIndex(indexName),
6158
client.Search.WithBody(&buf),
6259
)
6360
if err != nil {
@@ -79,12 +76,12 @@ func (p *ElasticSearchPlugin) GetGraph(config *engine.PluginConfig, database str
7976
doc := hits[0].(map[string]interface{})["_source"].(map[string]interface{})
8077

8178
for key := range doc {
82-
for _, otherCollection := range collections {
83-
singularName := strings.TrimSuffix(otherCollection, "s")
84-
if key == singularName+"_id" || key == otherCollection+"_id" {
79+
for otherIndexName := range indicesStats {
80+
singularName := strings.TrimSuffix(otherIndexName, "s")
81+
if key == singularName+"_id" || key == otherIndexName+"_id" {
8582
relations = append(relations, tableRelation{
86-
Table1: collectionName,
87-
Table2: otherCollection,
83+
Table1: indexName,
84+
Table2: otherIndexName,
8885
Relation: "ManyToMany",
8986
})
9087
}

‎core/src/plugins/elasticsearch/update.go

Lines changed: 38 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,13 @@ import (
88
"fmt"
99

1010
"github.com/clidey/whodb/core/src/engine"
11-
"github.com/elastic/go-elasticsearch/esapi"
1211
)
1312

13+
type JsonSourceMap struct {
14+
Id string `json:"_id"`
15+
Source json.RawMessage `json:"source"`
16+
}
17+
1418
func (p *ElasticSearchPlugin) UpdateStorageUnit(config *engine.PluginConfig, database string, storageUnit string, values map[string]string) (bool, error) {
1519
client, err := DB(config)
1620
if err != nil {
@@ -22,38 +26,49 @@ func (p *ElasticSearchPlugin) UpdateStorageUnit(config *engine.PluginConfig, dat
2226
return false, errors.New("missing 'document' key in values map")
2327
}
2428

25-
var jsonValues map[string]interface{}
26-
if err := json.Unmarshal([]byte(documentJSON), &jsonValues); err != nil {
27-
return false, err
28-
}
29-
30-
id, ok := jsonValues["_id"]
31-
if !ok {
32-
return false, errors.New("missing '_id' field in the document")
29+
jsonSourceMap := &JsonSourceMap{}
30+
if err := json.Unmarshal([]byte(documentJSON), jsonSourceMap); err != nil {
31+
return false, errors.New("source is not correctly formatted")
3332
}
3433

35-
idStr, ok := id.(string)
36-
if !ok {
37-
return false, errors.New("invalid '_id' field; not a valid string")
34+
var jsonValues map[string]interface{}
35+
if err := json.Unmarshal(jsonSourceMap.Source, &jsonValues); err != nil {
36+
return false, err
3837
}
3938

40-
delete(jsonValues, "_id")
39+
script := `
40+
for (entry in params.entrySet()) {
41+
ctx._source[entry.getKey()] = entry.getValue();
42+
}
43+
for (key in ctx._source.keySet().toArray()) {
44+
if (!params.containsKey(key)) {
45+
ctx._source.remove(key);
46+
}
47+
}
48+
`
49+
params := jsonValues
4150

4251
var buf bytes.Buffer
43-
if err := json.NewEncoder(&buf).Encode(jsonValues); err != nil {
52+
if err := json.NewEncoder(&buf).Encode(map[string]interface{}{
53+
"script": map[string]interface{}{
54+
"source": script,
55+
"lang": "painless",
56+
"params": params,
57+
},
58+
"upsert": jsonValues,
59+
}); err != nil {
4460
return false, err
4561
}
4662

47-
req := esapi.UpdateRequest{
48-
Index: storageUnit,
49-
DocumentID: idStr,
50-
Body: &buf,
51-
Refresh: "true",
52-
}
53-
54-
res, err := req.Do(context.Background(), client)
63+
res, err := client.Update(
64+
storageUnit,
65+
jsonSourceMap.Id,
66+
&buf,
67+
client.Update.WithContext(context.Background()),
68+
client.Update.WithRefresh("true"),
69+
)
5570
if err != nil {
56-
return false, err
71+
return false, fmt.Errorf("failed to execute update: %w", err)
5772
}
5873
defer res.Body.Close()
5974

‎dev/sample-data/elasticsearch/data.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{ "index": { "_index": "sample_data", "_id": "1" } }
2+
{ "name": "John Doe", "age": 30, "occupation": "Software Engineer" }
3+
{ "index": { "_index": "sample_data", "_id": "2" } }
4+
{ "name": "Jane Smith", "age": 25, "occupation": "Data Scientist" }
5+
{ "index": { "_index": "sample_data2", "_id": "1" } }
6+
{ "name": "John Doe", "age": 30, "occupation": "Software Engineer" }
7+
{ "index": { "_index": "sample_data2", "_id": "2" } }
8+
{ "name": "Jane Smith", "age": 25, "occupation": "Data Scientist" }
9+
{ "index": { "_index": "sample_data3", "_id": "1" } }
10+
{ "name": "John Doe", "age": 30, "occupation": "Software Engineer" }
11+
{ "index": { "_index": "sample_data3", "_id": "2" } }
12+
{ "name": "Jane Smith", "age": 25, "occupation": "Data Scientist", "sample_data3_id": "2" }
13+
14+
15+

‎dev/sample-data/elasticsearch/upload.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
curl -XPOST "http://localhost:9200/_bulk" -H "Content-Type: application/json" --data-binary @data.json
2+
3+
4+
# where clause {"query":{"query_string":{"query":"*John*"}}}

‎frontend/src/components/sidebar/sidebar.tsx

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import { AuthActions, LoginProfile } from "../../store/auth";
1313
import { DatabaseActions } from "../../store/database";
1414
import { notify } from "../../store/function";
1515
import { useAppSelector } from "../../store/hooks";
16-
import { createStub, isNoSQL } from "../../utils/functions";
16+
import { createStub, getDatabaseStorageUnitLabel } from "../../utils/functions";
1717
import { AnimatedButton } from "../button";
1818
import { BRAND_COLOR } from "../classes";
1919
import { Dropdown, IDropdownItem } from "../dropdown";
@@ -126,6 +126,9 @@ function getDropdownLoginProfileItem(profile: LoginProfile): IDropdownItem {
126126
};
127127
}
128128

129+
const DATABASES_THAT_SUPPORT_SCRATCH_PAD = [DatabaseType.MongoDb, DatabaseType.Redis, DatabaseType.ElasticSearch];
130+
const DATABASES_THAT_DONT_SUPPORT_SCHEMA = [DatabaseType.Sqlite3, DatabaseType.Redis, DatabaseType.ElasticSearch];
131+
129132
export const Sidebar: FC = () => {
130133
const [collapsed, setCollapsed] = useState(false);
131134
const schema = useAppSelector(state => state.database.schema);
@@ -149,7 +152,7 @@ export const Sidebar: FC = () => {
149152
}, [dispatch, navigate, pathname]);
150153

151154
useEffect(() => {
152-
if (current == null || [DatabaseType.Sqlite3, DatabaseType.Redis].includes(current?.Type as DatabaseType)) {
155+
if (current == null || DATABASES_THAT_DONT_SUPPORT_SCHEMA.includes(current?.Type as DatabaseType)) {
153156
return;
154157
}
155158
if (schema === "") {
@@ -176,7 +179,7 @@ export const Sidebar: FC = () => {
176179
}
177180
const routes = [
178181
{
179-
title: isNoSQL(current.Type) ? "Collections" : "Tables",
182+
title: getDatabaseStorageUnitLabel(current.Type),
180183
icon: Icons.Tables,
181184
path: InternalRoutes.Dashboard.StorageUnit.path,
182185
},
@@ -186,7 +189,7 @@ export const Sidebar: FC = () => {
186189
path: InternalRoutes.Graph.path,
187190
},
188191
];
189-
if (current.Type !== DatabaseType.MongoDb && current.Type !== DatabaseType.Redis) {
192+
if (!DATABASES_THAT_SUPPORT_SCRATCH_PAD.includes(current.Type as DatabaseType)) {
190193
routes.push({
191194
title: "Scratchpad",
192195
icon: Icons.Console,
@@ -345,7 +348,7 @@ export const Sidebar: FC = () => {
345348
{
346349
data != null &&
347350
<div className={classNames("flex gap-2 items-center w-full", {
348-
"hidden": pathname === InternalRoutes.RawExecute.path || collapsed || [DatabaseType.Sqlite3, DatabaseType.Redis].includes(current?.Type as DatabaseType),
351+
"hidden": pathname === InternalRoutes.RawExecute.path || collapsed || DATABASES_THAT_DONT_SUPPORT_SCHEMA.includes(current?.Type as DatabaseType),
349352
})}>
350353
<div className="text-sm text-gray-600">Schema:</div>
351354
<Dropdown className="w-[140px]" value={{ id: schema, label: schema }} items={data.Schema.map(schema => ({ id: schema, label: schema }))} onChange={handleSchemaChange}

‎frontend/src/pages/auth/login.tsx

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ const databaseTypeDropdownItems: IDropdownItem<Record<string, string>>[] = [
5151
id: "ElasticSearch",
5252
label: "ElasticSearch",
5353
icon: Icons.Logos.ElasticSearch,
54+
extra: {"Port": "9200", "SSL Mode": "disable"},
5455
},
5556
]
5657

@@ -170,7 +171,7 @@ export const LoginPage: FC = () => {
170171
<InputWithlabel label="Host Name" value={hostName} setValue={setHostName} />
171172
{ databaseType.id !== DatabaseType.Redis && <InputWithlabel label="Username" value={username} setValue={setUsername} /> }
172173
<InputWithlabel label="Password" value={password} setValue={setPassword} type="password" />
173-
{ (databaseType.id !== DatabaseType.MongoDb && databaseType.id !== DatabaseType.Redis) && <InputWithlabel label="Database" value={database} setValue={setDatabase} /> }
174+
{ (databaseType.id !== DatabaseType.MongoDb && databaseType.id !== DatabaseType.Redis && databaseType.id !== DatabaseType.ElasticSearch) && <InputWithlabel label="Database" value={database} setValue={setDatabase} /> }
174175
</>
175176
}, [database, databaseType.id, databasesLoading, foundDatabases?.Database, hostName, password, username]);
176177

0 commit comments

Comments
 (0)