From f2307bfeb6f06e2236f3b8c504aea44ed35d5de4 Mon Sep 17 00:00:00 2001 From: egisz Date: Tue, 5 May 2020 14:23:41 +0300 Subject: [PATCH 01/25] Add text filter to logs page --- src/modules/repository/logs_repository.js | 21 +++++++----- src/modules/system/logs_http.js | 24 +++++++------ templates/logs.html.twig | 42 ++++++++++++++++------- 3 files changed, 56 insertions(+), 31 deletions(-) diff --git a/src/modules/repository/logs_repository.js b/src/modules/repository/logs_repository.js index 7b70db820..00a1f0941 100644 --- a/src/modules/repository/logs_repository.js +++ b/src/modules/repository/logs_repository.js @@ -5,22 +5,27 @@ module.exports = class LogsRepository { this.db = db; } - getLatestLogs(excludes = ['debug'], limit = 200) { + getLatestLogs(filters = { logExcludeLevels: ['debug'], logTxtFilter: '' }, limit = 200) { return new Promise(resolve => { - let sql = `SELECT * from logs order by created_at DESC LIMIT ${limit}`; - const parameters = {}; + let sql = `SELECT * from logs WHERE 1=1`; - if (excludes.length > 0) { - sql = `SELECT * from logs WHERE level NOT IN (${excludes - .map((exclude, index) => `$level_${index}`) - .join(', ')}) order by created_at DESC LIMIT ${limit}`; + if (filters.logExcludeLevels.length > 0) { + sql += ` AND level NOT IN (${filters.logExcludeLevels + .map((_exclude, index) => `$level_${index}`) + .join(', ')}) `; - excludes.forEach((exclude, index) => { + filters.logExcludeLevels.forEach((exclude, index) => { parameters[`level_${index}`] = exclude; }); } + if (filters.logTxtFilter !== '') { + sql += ` AND message LIKE '%${filters.logTxtFilter}%'`; + } + + sql += ` order by created_at DESC LIMIT ${limit}`; + const stmt = this.db.prepare(sql); resolve(stmt.all(parameters)); }); diff --git a/src/modules/system/logs_http.js b/src/modules/system/logs_http.js index 2bc2b3274..7a4d87a7d 100644 --- a/src/modules/system/logs_http.js +++ b/src/modules/system/logs_http.js @@ -1,26 +1,30 @@ -const _ = require('lodash'); - module.exports = class LogsHttp { constructor(logsRepository) { this.logsRepository = logsRepository; } async getLogsPageVariables(request, response) { - let excludeLevels = request.query.exclude_levels || []; - - if (excludeLevels.length === 0 && !('filters' in request.cookies)) { - excludeLevels = ['debug']; - } + const filters = request.query.filters || + request.cookies.filters || { logExcludeLevels: ['debug'], logTxtFilter: '' }; - response.cookie('filters', excludeLevels, { + response.cookie('filters', filters, { maxAge: 60 * 60 * 24 * 30 * 1000 }); + // We will highlight profit number in logs output + const logs = await this.logsRepository.getLatestLogs(filters); + logs.forEach(log => { + const profit = log.message.match(/(?<=profit":)(-?\d+.\d+)/); + if (profit) { + [ log.profit ] = profit; + } + }); + return { - logs: await this.logsRepository.getLatestLogs(excludeLevels), + logs: logs, levels: await this.logsRepository.getLevels(), form: { - excludeLevels: excludeLevels + filters: filters } }; } diff --git a/templates/logs.html.twig b/templates/logs.html.twig index fb3f5a58a..1852dab07 100644 --- a/templates/logs.html.twig +++ b/templates/logs.html.twig @@ -32,18 +32,26 @@
Filters
- -
    -
  • Exclude:
    • - {% for key,level in levels %} -
  • -
    - - -
    -
    • - {% endfor %} -
+
+
+
    +
  • Exclude:
    • + {% for key,level in levels %} +
  • +
    + + +
    +
    • + {% endfor %} +
+
+
+ + +
+ +
@@ -70,7 +78,15 @@ {{ log.level }} {% endif %} - {{ log.message|escape('html') }} + + {% if log.profit and log.profit >= 0 %} + {{ log.message|replace({(log.profit): ('' ~ log.profit ~ '') }) }} + {% elseif log.profit and log.profit < 0 %} + {{ log.message|replace({(log.profit): ('' ~ log.profit ~ '') }) }} + {% else %} + {{ log.message }} + {% endif %} + {{ log.created_at|date('y-m-d H:i:s') }} {% endfor %} From 8574a2b99160072a647c3bb5305452688a6a2878 Mon Sep 17 00:00:00 2001 From: egisz Date: Mon, 11 May 2020 15:49:59 +0300 Subject: [PATCH 02/25] use datatables in logs page start of work --- src/modules/http.js | 5 +++++ src/modules/system/logs_http.js | 29 ++++++++++++++++++++++++ templates/logs.html.twig | 40 ++++++++++++--------------------- web/static/js/logs.js | 32 ++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 26 deletions(-) create mode 100644 web/static/js/logs.js diff --git a/src/modules/http.js b/src/modules/http.js index d5f6deeb6..1bbcc1220 100644 --- a/src/modules/http.js +++ b/src/modules/http.js @@ -152,6 +152,11 @@ module.exports = class Http { res.render('../templates/logs.html.twig', await this.logsHttp.getLogsPageVariables(req, res)); }); + app.post('/logsTable', async (req, res) => { + const logs = await this.logsHttp.getLogsData(req, res); + res.json(logs); + }); + app.get('/desks/:desk', async (req, res) => { res.render('../templates/desks.html.twig', { desk: this.systemUtil.getConfig('desks')[req.params.desk], diff --git a/src/modules/system/logs_http.js b/src/modules/system/logs_http.js index 7a4d87a7d..bf51dd3f0 100644 --- a/src/modules/system/logs_http.js +++ b/src/modules/system/logs_http.js @@ -28,4 +28,33 @@ module.exports = class LogsHttp { } }; } + + async getLogsData2(request, response) { + + // We will highlight profit number in logs output + const logs = await this.logsRepository.getFilteredLogs(request.body); + logs.forEach(log => { + const profit = log.message.match(/(?<=profit":)(-?\d+.\d+)/); + if (profit) { + [ log.profit ] = profit; + } + }); + + const recordsTotal = await this.logsRepository.getTotal(); + return { + draw: request.body.draw, + recordsFiltered: logs.length, + recordsTotal: recordsTotal.total, + data: logs + }; + } + + async getLogsData(request, response){ + datatable(this.logsRepository, reqest.query, {}) + .then((result) => { + // result is response for datatables + return result; + }); + + } }; diff --git a/templates/logs.html.twig b/templates/logs.html.twig index 1852dab07..4f4a51da1 100644 --- a/templates/logs.html.twig +++ b/templates/logs.html.twig @@ -2,6 +2,11 @@ {% block title %}Logs | Crypto Bot{% endblock %} +{% block head %} + +{% endblock %} + + {% block content %}
@@ -58,7 +63,7 @@
- +
@@ -66,33 +71,10 @@ - - {% for log in logs %} - - - - - - {% endfor %} - +
LevelCreatedAt
- {% if log.level == 'debug' %} - {{ log.level }} - {% elseif log.level == 'info' %} - {{ log.level }} - {% elseif log.level == 'error' %} - {{ log.level }} - {% endif %} - - {% if log.profit and log.profit >= 0 %} - {{ log.message|replace({(log.profit): ('' ~ log.profit ~ '') }) }} - {% elseif log.profit and log.profit < 0 %} - {{ log.message|replace({(log.profit): ('' ~ log.profit ~ '') }) }} - {% else %} - {{ log.message }} - {% endif %} - {{ log.created_at|date('y-m-d H:i:s') }}
+
@@ -105,3 +87,9 @@ {% endblock %} + +{% block javascript %} +{{ parent() }} + + +{% endblock javascript %} diff --git a/web/static/js/logs.js b/web/static/js/logs.js new file mode 100644 index 000000000..e0a895744 --- /dev/null +++ b/web/static/js/logs.js @@ -0,0 +1,32 @@ +$(function() { + const t = $('#logsTable').DataTable({ + paging: true, + pageLength: 10, + processing: true, + serverSide: true, + ajax: { + type: 'POST', + url: '/logsTable' + }, + // data: [{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.6820223348452448,\"updatedAt\":\"2020-05-06T09:46:55.903Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:46:55.903Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758406\",\"bid\":9078.8,\"ask\":9078.9,\"createdAt\":\"2020-05-06T09:46:46.151Z\"}","created_at":1588758423,"profit":"0.6820223348452448"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.6820223348452448,\"updatedAt\":\"2020-05-06T09:46:25.905Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:46:25.905Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758376\",\"bid\":9078.8,\"ask\":9078.9,\"createdAt\":\"2020-05-06T09:46:16.119Z\"}","created_at":1588758393,"profit":"0.6820223348452448"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.6565158085014344,\"updatedAt\":\"2020-05-06T09:45:55.912Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:45:55.912Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758356\",\"bid\":9076.5,\"ask\":9077,\"createdAt\":\"2020-05-06T09:45:56.173Z\"}","created_at":1588758362,"profit":"0.6565158085014344"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.6609517261264575,\"updatedAt\":\"2020-05-06T09:45:25.900Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:45:25.900Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758326\",\"bid\":9076.9,\"ask\":9077,\"createdAt\":\"2020-05-06T09:45:26.161Z\"}","created_at":1588758331,"profit":"0.6609517261264575"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.6110476528450981,\"updatedAt\":\"2020-05-06T09:44:55.900Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:44:55.900Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758296\",\"bid\":9072.4,\"ask\":9072.5,\"createdAt\":\"2020-05-06T09:44:56.121Z\"}","created_at":1588758300,"profit":"0.6110476528450981"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.5811052088762869,\"updatedAt\":\"2020-05-06T09:44:25.906Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:44:25.906Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758266\",\"bid\":9069.7,\"ask\":9069.8,\"createdAt\":\"2020-05-06T09:44:26.124Z\"}","created_at":1588758269,"profit":"0.5811052088762869"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.5633615383762391,\"updatedAt\":\"2020-05-06T09:43:55.905Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:43:55.905Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758236\",\"bid\":9068.1,\"ask\":9069.7,\"createdAt\":\"2020-05-06T09:43:56.171Z\"}","created_at":1588758239,"profit":"0.5633615383762391"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.5844321470950264,\"updatedAt\":\"2020-05-06T09:43:25.896Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:43:25.896Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758196\",\"bid\":9070,\"ask\":9070.1,\"createdAt\":\"2020-05-06T09:43:16.121Z\"}","created_at":1588758208,"profit":"0.5844321470950264"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.6032847970013355,\"updatedAt\":\"2020-05-06T09:42:55.897Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:42:55.897Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758146\",\"bid\":9071.7,\"ask\":9071.8,\"createdAt\":\"2020-05-06T09:42:26.187Z\"}","created_at":1588758177,"profit":"0.6032847970013355"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.6032847970013355,\"updatedAt\":\"2020-05-06T09:42:25.894Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:42:25.894Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758146\",\"bid\":9071.7,\"ask\":9071.8,\"createdAt\":\"2020-05-06T09:42:26.187Z\"}","created_at":1588758146,"profit":"0.6032847970013355"}], + columns: [ + { data: 'level', name: 'Level' }, + { data: 'message', name: 'Message' }, + { + data: 'created_at', + name: 'CreatedAt' + /* , + formatter: function($d, $row) { + return date('jS M y', strtotime($d)); + } */ + } + ], + columnDefs: [ + { + searchable: true, + orderable: true, + targets: 0 + } + ] + }); +}); From ca845ffd4423ef883c78e23374846c1dabd8e340 Mon Sep 17 00:00:00 2001 From: egisz Date: Mon, 11 May 2020 15:55:30 +0300 Subject: [PATCH 03/25] sequelize semi working --- .sequelizerc | 10 + conf.json.dist | 18 +- package-lock.json | 512 +++++++++++++++++- package.json | 7 +- .../listener/ticker_database_listener.js | 15 +- .../repository/candlestick_repository.js | 174 +++--- src/modules/repository/logs_repository.js | 54 -- src/modules/repository/signal_repository.js | 75 ++- .../repository/ticker_log_repository.js | 21 - src/modules/repository/ticker_repository.js | 61 ++- src/modules/services.js | 105 ++-- src/modules/trade.js | 6 +- src/utils/winston_sqlite_transport.js | 48 -- 13 files changed, 775 insertions(+), 331 deletions(-) create mode 100644 .sequelizerc delete mode 100644 src/modules/repository/logs_repository.js delete mode 100644 src/modules/repository/ticker_log_repository.js delete mode 100644 src/utils/winston_sqlite_transport.js diff --git a/.sequelizerc b/.sequelizerc new file mode 100644 index 000000000..6383033bf --- /dev/null +++ b/.sequelizerc @@ -0,0 +1,10 @@ +// .sequelizerc + +const path = require('path'); + +module.exports = { + 'config': path.resolve('conf.json'), + 'models-path': path.resolve('src', 'modules', 'repository'), + 'seeders-path': path.resolve('db', 'seeders'), + 'migrations-path': path.resolve('db', 'migrations') +}; \ No newline at end of file diff --git a/conf.json.dist b/conf.json.dist index eaf06f99b..fc7e4f2a0 100644 --- a/conf.json.dist +++ b/conf.json.dist @@ -147,5 +147,21 @@ } ] } - ] + ], + + "development": { + "storage": "bot.db", + "dialect": "sqlite", + "operatorsAliases": false + }, + "test": { + "storage": "bot.db", + "dialect": "sqlite", + "operatorsAliases": false + }, + "production": { + "storage": "bot.db", + "dialect": "sqlite", + "operatorsAliases": false + } } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 47d12273c..bbf74b431 100644 --- a/package-lock.json +++ b/package-lock.json @@ -234,6 +234,11 @@ "form-data": "*" } }, + "@types/geojson": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-1.0.6.tgz", + "integrity": "sha512-Xqg/lIZMrUd0VRmSRbCAewtwGZiAk3mEUDvV4op1tGl+LvyPcb/MIOSxTl9z+9+J+R4/vpjiCAT4xeKzH9ji1w==" + }, "@types/node": { "version": "12.12.17", "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.17.tgz", @@ -347,6 +352,11 @@ "resolved": "https://registry.npmjs.org/ansi-wrap/-/ansi-wrap-0.1.0.tgz", "integrity": "sha1-qCJQ3bABXponyoLoLqYDu/pF768=" }, + "any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=" + }, "anymatch": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", @@ -1141,15 +1151,6 @@ "resolved": "https://registry.npmjs.org/better-queue-memory/-/better-queue-memory-1.0.4.tgz", "integrity": "sha512-SWg5wFIShYffEmJpI6LgbL8/3Dqhku7xI1oEiy6FroP9DbcZlG0ZDjxvPdP9t7hTGW40IpIcC6zVoGT1oxjOuA==" }, - "better-sqlite3": { - "version": "5.4.3", - "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-5.4.3.tgz", - "integrity": "sha512-fPp+8f363qQIhuhLyjI4bu657J/FfMtgiiHKfaTsj3RWDkHlWC1yT7c6kHZDnBxzQVoAINuzg553qKmZ4F1rEw==", - "requires": { - "integer": "^2.1.0", - "tar": "^4.4.10" - } - }, "bfx-api-node-models": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/bfx-api-node-models/-/bfx-api-node-models-1.1.2.tgz", @@ -1679,6 +1680,19 @@ "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-1.0.0.tgz", "integrity": "sha1-T6kXw+WclKAEzWH47lCdplFocUM=" }, + "cli-color": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/cli-color/-/cli-color-1.4.0.tgz", + "integrity": "sha512-xu6RvQqqrWEo6MPR1eixqGPywhYBHRs653F9jfXB2Hx4jdM/3WxiNE1vppRmxtMIfl16SFYTpYlrnqH/HsK/2w==", + "requires": { + "ansi-regex": "^2.1.1", + "d": "1", + "es5-ext": "^0.10.46", + "es6-iterator": "^2.0.3", + "memoizee": "^0.4.14", + "timers-ext": "^0.1.5" + } + }, "cli-cursor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", @@ -1739,6 +1753,15 @@ "resolved": "https://registry.npmjs.org/clone-stats/-/clone-stats-0.0.1.tgz", "integrity": "sha1-uI+UqCzzi4eR1YBG6kAprYjKmdE=" }, + "cls-bluebird": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cls-bluebird/-/cls-bluebird-2.1.0.tgz", + "integrity": "sha1-N+8eCAqP+1XC9BZPU28ZGeeWiu4=", + "requires": { + "is-bluebird": "^1.0.2", + "shimmer": "^1.1.0" + } + }, "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", @@ -1878,6 +1901,15 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, + "config-chain": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.12.tgz", + "integrity": "sha512-a1eOIcu8+7lUInge4Rpf/n4Krkf3Dd9lqhljRzII1/Zno/kRtUWnznPO3jOKBmTEktkt3fkxisUcivoj0ebzoA==", + "requires": { + "ini": "^1.3.4", + "proto-list": "~1.2.1" + } + }, "configstore": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/configstore/-/configstore-3.1.2.tgz", @@ -2021,6 +2053,15 @@ "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-1.0.0.tgz", "integrity": "sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4=" }, + "d": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", + "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", + "requires": { + "es5-ext": "^0.10.50", + "type": "^1.0.1" + } + }, "damerau-levenshtein": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.5.tgz", @@ -2035,6 +2076,23 @@ "assert-plus": "^1.0.0" } }, + "datatables.net": { + "version": "1.10.20", + "resolved": "https://registry.npmjs.org/datatables.net/-/datatables.net-1.10.20.tgz", + "integrity": "sha512-4E4S7tTU607N3h0fZPkGmAtr9mwy462u+VJ6gxYZ8MxcRIjZqHy3Dv1GNry7i3zQCktTdWbULVKBbkAJkuHEnQ==", + "requires": { + "jquery": ">=1.7" + } + }, + "datatables.net-bs4": { + "version": "1.10.20", + "resolved": "https://registry.npmjs.org/datatables.net-bs4/-/datatables.net-bs4-1.10.20.tgz", + "integrity": "sha512-kQmMUMsHMOlAW96ztdoFqjSbLnlGZQ63iIM82kHbmldsfYdzuyhbb4hTx6YNBi481WCO3iPSvI6YodNec46ZAw==", + "requires": { + "datatables.net": "1.10.20", + "jquery": ">=1.7" + } + }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -2145,6 +2203,11 @@ "is-obj": "^1.0.0" } }, + "dottie": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.2.tgz", + "integrity": "sha512-fmrwR04lsniq/uSr8yikThDTrM7epXHBAAjH9TbeH3rEA8tdCO7mRzB9hdmdGyJCxF8KERo9CITcm3kGuoyMhg==" + }, "duplexer3": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", @@ -2159,6 +2222,24 @@ "safer-buffer": "^2.1.0" } }, + "editorconfig": { + "version": "0.15.3", + "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz", + "integrity": "sha512-M9wIMFx96vq0R4F+gRpY3o2exzb8hEj/n9S8unZtHSvYjibBp/iMufSzvmOcV/laG0ZtuTVGtiJggPOSW2r93g==", + "requires": { + "commander": "^2.19.0", + "lru-cache": "^4.1.5", + "semver": "^5.6.0", + "sigmund": "^1.0.1" + }, + "dependencies": { + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + } + } + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -2245,11 +2326,51 @@ "is-symbol": "^1.0.2" } }, + "es5-ext": { + "version": "0.10.53", + "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz", + "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==", + "requires": { + "es6-iterator": "~2.0.3", + "es6-symbol": "~3.1.3", + "next-tick": "~1.0.0" + } + }, + "es6-iterator": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", + "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=", + "requires": { + "d": "1", + "es5-ext": "^0.10.35", + "es6-symbol": "^3.1.1" + } + }, "es6-promise": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==" }, + "es6-symbol": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", + "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", + "requires": { + "d": "^1.0.1", + "ext": "^1.1.2" + } + }, + "es6-weak-map": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.3.tgz", + "integrity": "sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==", + "requires": { + "d": "1", + "es5-ext": "^0.10.46", + "es6-iterator": "^2.0.3", + "es6-symbol": "^3.1.1" + } + }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", @@ -3270,6 +3391,15 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, + "event-emitter": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", + "integrity": "sha1-34xp7vFkeSPHFXuc6DhAYQsCzDk=", + "requires": { + "d": "1", + "es5-ext": "~0.10.14" + } + }, "eventemitter2": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-5.0.1.tgz", @@ -3382,6 +3512,21 @@ } } }, + "ext": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", + "integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==", + "requires": { + "type": "^2.0.0" + }, + "dependencies": { + "type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/type/-/type-2.0.0.tgz", + "integrity": "sha512-KBt58xCHry4Cejnc2ISQAF7QY+ORngsWfxezO68+12hKV6lQY8P/psIkcbjeHWn7MqcgciWJyCCevFMJdIXpow==" + } + } + }, "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -3777,6 +3922,11 @@ } } }, + "generic-pool": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.5.0.tgz", + "integrity": "sha512-dEkxmX+egB2o4NR80c/q+xzLLzLX+k68/K8xv81XprD+Sk7ZtP14VugeCz+fUwv5FzpWq40pPtAkzPRqT8ka9w==" + }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -4072,6 +4222,11 @@ "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=" }, + "inflection": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.12.0.tgz", + "integrity": "sha1-ogCTVlbW9fa8TcdQLhrstwMihBY=" + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -4200,11 +4355,6 @@ } } }, - "integer": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/integer/-/integer-2.1.0.tgz", - "integrity": "sha512-vBtiSgrEiNocWvvZX1RVfeOKa2mCHLZQ2p9nkQkQZ/BvEiY+6CcUz0eyjvIiewjJoeNidzg2I+tpPJvpyspL1w==" - }, "invariant": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", @@ -4258,6 +4408,11 @@ "binary-extensions": "^2.0.0" } }, + "is-bluebird": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-bluebird/-/is-bluebird-1.0.2.tgz", + "integrity": "sha1-CWQ5Bg9KpBGr7hkUOoTWpVNG1uI=" + }, "is-buffer": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", @@ -4395,8 +4550,7 @@ "is-promise": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", - "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=", - "dev": true + "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=" }, "is-redirect": { "version": "1.0.0", @@ -4503,6 +4657,39 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "jquery": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.5.1.tgz", + "integrity": "sha512-XwIBPqcMn57FxfT+Go5pzySnm4KWkT1Tv7gjrpT1srtf8Weynl6R273VJ5GjkRb51IzMp5nbaPjJXMWeju2MKg==" + }, + "js-beautify": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.11.0.tgz", + "integrity": "sha512-a26B+Cx7USQGSWnz9YxgJNMmML/QG2nqIaL7VVYPCXbqiKz8PN0waSNvroMtvAK6tY7g/wPdNWGEP+JTNIBr6A==", + "requires": { + "config-chain": "^1.1.12", + "editorconfig": "^0.15.3", + "glob": "^7.1.3", + "mkdirp": "~1.0.3", + "nopt": "^4.0.3" + }, + "dependencies": { + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" + }, + "nopt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", + "requires": { + "abbrev": "1", + "osenv": "^0.1.4" + } + } + } + }, "js-tokens": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz", @@ -4796,6 +4983,14 @@ } } }, + "lru-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/lru-queue/-/lru-queue-0.1.0.tgz", + "integrity": "sha1-Jzi9nw089PhEkMVzbEhpmsYyzaM=", + "requires": { + "es5-ext": "~0.10.2" + } + }, "mailcomposer": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/mailcomposer/-/mailcomposer-3.12.0.tgz", @@ -4864,6 +5059,21 @@ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" }, + "memoizee": { + "version": "0.4.14", + "resolved": "https://registry.npmjs.org/memoizee/-/memoizee-0.4.14.tgz", + "integrity": "sha512-/SWFvWegAIYAO4NQMpcX+gcra0yEZu4OntmUdrBaWrJncxOqAziGFlHxc7yjKVK2uu3lpPW27P27wkR82wA8mg==", + "requires": { + "d": "1", + "es5-ext": "^0.10.45", + "es6-weak-map": "^2.0.2", + "event-emitter": "^0.3.5", + "is-promise": "^2.1", + "lru-queue": "0.1", + "next-tick": "1", + "timers-ext": "^0.1.5" + } + }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -5121,6 +5331,14 @@ "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" }, + "moment-timezone": { + "version": "0.5.28", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.28.tgz", + "integrity": "sha512-TDJkZvAyKIVWg5EtVqRzU97w0Rb0YVbfpqyjgu6GwXCAohVRqwZjf4fOzDE6p1Ch98Sro/8hQQi65WDXW5STPw==", + "requires": { + "moment": ">= 2.9.0" + } + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -5260,6 +5478,11 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" }, + "next-tick": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", + "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" + }, "nice-try": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", @@ -5772,8 +5995,7 @@ "path-parse": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", - "dev": true + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==" }, "path-to-regexp": { "version": "0.1.7", @@ -5939,6 +6161,11 @@ "react-is": "^16.8.1" } }, + "proto-list": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", + "integrity": "sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=" + }, "proxy-addr": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz", @@ -6283,7 +6510,6 @@ "version": "1.13.1", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.13.1.tgz", "integrity": "sha512-CxqObCX8K8YtAhOBRg+lrcdn+LK+WYOS8tSjqSFbjtrI5PnS63QPhZl4+yKfrU9tdsbMu9Anr/amegT87M9Z6w==", - "dev": true, "requires": { "path-parse": "^1.0.6" } @@ -6323,6 +6549,14 @@ "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==" }, + "retry-as-promised": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-3.2.0.tgz", + "integrity": "sha512-CybGs60B7oYU/qSQ6kuaFmRd9sTZ6oXSc0toqePvV74Ac6/IFZSI1ReFQmtCN+uvW1Mtqdwpvt/LGOiCBAY2Mg==", + "requires": { + "any-promise": "^1.3.0" + } + }, "rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", @@ -6431,6 +6665,141 @@ "mailcomposer": "3.12.0" } }, + "sequelize": { + "version": "5.21.7", + "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-5.21.7.tgz", + "integrity": "sha512-+JrS5Co7CN53cOFFFaUb+xqQP00wD1Ag9xGLBLoUko2KhRZxjm+UDkqAVPHTUp87McLwJaCPkKv61GPhBVloRQ==", + "requires": { + "bluebird": "^3.5.0", + "cls-bluebird": "^2.1.0", + "debug": "^4.1.1", + "dottie": "^2.0.0", + "inflection": "1.12.0", + "lodash": "^4.17.15", + "moment": "^2.24.0", + "moment-timezone": "^0.5.21", + "retry-as-promised": "^3.2.0", + "semver": "^6.3.0", + "sequelize-pool": "^2.3.0", + "toposort-class": "^1.0.1", + "uuid": "^3.3.3", + "validator": "^10.11.0", + "wkx": "^0.4.8" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, + "sequelize-cli": { + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/sequelize-cli/-/sequelize-cli-5.5.1.tgz", + "integrity": "sha512-ZM4kUZvY3y14y+Rq3cYxGH7YDJz11jWHcN2p2x7rhAIemouu4CEXr5ebw30lzTBtyXV4j2kTO+nUjZOqzG7k+Q==", + "requires": { + "bluebird": "^3.5.3", + "cli-color": "^1.4.0", + "fs-extra": "^7.0.1", + "js-beautify": "^1.8.8", + "lodash": "^4.17.5", + "resolve": "^1.5.0", + "umzug": "^2.1.0", + "yargs": "^13.1.0" + } + }, + "sequelize-datatable": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/sequelize-datatable/-/sequelize-datatable-2.0.0.tgz", + "integrity": "sha512-OAc31wVkbEyo24kJpZ2DP+X4HGAcsGxiGQIvuoqlRHiXYcgPJu+i+sSQlL3Pd43cG3Jr8e3JpId+8OgPtCbLJQ==", + "requires": { + "bluebird": "^3.5.1", + "lodash": "^4.17.10", + "sequelize": "^4.37.10" + }, + "dependencies": { + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "retry-as-promised": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-2.3.2.tgz", + "integrity": "sha1-zZdO5P2bX+A8vzGHHuSCIcB3N7c=", + "requires": { + "bluebird": "^3.4.6", + "debug": "^2.6.9" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } + } + }, + "sequelize": { + "version": "4.44.4", + "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-4.44.4.tgz", + "integrity": "sha512-nkHmYkbwQK7uwpgW9VBalCBnQqQ8mslTdgcBthtJLORuPvAYRPlfkXZMVUU9TLLJt9CX+/y0MYg0DpcP6ywsEQ==", + "requires": { + "bluebird": "^3.5.0", + "cls-bluebird": "^2.1.0", + "debug": "^3.1.0", + "depd": "^1.1.0", + "dottie": "^2.0.0", + "generic-pool": "3.5.0", + "inflection": "1.12.0", + "lodash": "^4.17.1", + "moment": "^2.20.0", + "moment-timezone": "^0.5.14", + "retry-as-promised": "^2.3.2", + "semver": "^5.5.0", + "terraformer-wkt-parser": "^1.1.2", + "toposort-class": "^1.0.1", + "uuid": "^3.2.1", + "validator": "^10.4.0", + "wkx": "^0.4.1" + } + } + } + }, + "sequelize-pool": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-2.3.0.tgz", + "integrity": "sha512-Ibz08vnXvkZ8LJTiUOxRcj1Ckdn7qafNZ2t59jYHMX1VIebTAOYefWdRYFt6z6+hy52WGthAHAoLc9hvk3onqA==" + }, "serve-static": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", @@ -6484,6 +6853,16 @@ "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=" }, + "shimmer": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", + "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" + }, + "sigmund": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz", + "integrity": "sha1-P/IfGYytIXX587eBhT/ZTQ0ZtZA=" + }, "signal-exit": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", @@ -6699,6 +7078,34 @@ "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" }, + "sqlite3": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-4.2.0.tgz", + "integrity": "sha512-roEOz41hxui2Q7uYnWsjMOTry6TcNUNmp8audCx18gF10P2NknwdpF+E+HKvz/F2NvPKGGBF4NGc+ZPQ+AABwg==", + "requires": { + "nan": "^2.12.1", + "node-pre-gyp": "^0.11.0" + }, + "dependencies": { + "node-pre-gyp": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz", + "integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==", + "requires": { + "detect-libc": "^1.0.2", + "mkdirp": "^0.5.1", + "needle": "^2.2.1", + "nopt": "^4.0.1", + "npm-packlist": "^1.1.6", + "npmlog": "^4.0.2", + "rc": "^1.2.7", + "rimraf": "^2.6.1", + "semver": "^5.3.0", + "tar": "^4" + } + } + } + }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", @@ -7034,6 +7441,23 @@ "execa": "^0.7.0" } }, + "terraformer": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/terraformer/-/terraformer-1.0.12.tgz", + "integrity": "sha512-MokUp0+MFal4CmJDVL6VAO1bKegeXcBM2RnPVfqcFIp2IIv8EbPAjG0j/vEy/vuKB8NVMMSF2vfpVS/QLe4DBg==", + "requires": { + "@types/geojson": "^7946.0.0 || ^1.0.0" + } + }, + "terraformer-wkt-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/terraformer-wkt-parser/-/terraformer-wkt-parser-1.2.1.tgz", + "integrity": "sha512-+CJyNLWb3lJ9RsZMTM66BY0MT3yIo4l4l22Jd9CrZuwzk54fsu4Sc7zejuS9fCITTuTQy3p06d4MZMVI7v5wSg==", + "requires": { + "@types/geojson": "^1.0.0", + "terraformer": "~1.0.5" + } + }, "text-hex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", @@ -7065,6 +7489,15 @@ "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=" }, + "timers-ext": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.7.tgz", + "integrity": "sha512-b85NUNzTSdodShTIbky6ZF02e8STtVVfD+fu4aXXShEELpozH+bCpJLYMPZbsABN2wDH7fJpqIoXxJpzbf0NqQ==", + "requires": { + "es5-ext": "~0.10.46", + "next-tick": "1" + } + }, "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", @@ -7240,6 +7673,11 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" }, + "toposort-class": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz", + "integrity": "sha1-f/0feMi+KMO6Rc1OGj9e4ZO9mYg=" + }, "tough-cookie": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", @@ -7327,6 +7765,11 @@ } } }, + "type": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", + "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" + }, "type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", @@ -7356,6 +7799,14 @@ "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz", "integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og==" }, + "umzug": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/umzug/-/umzug-2.3.0.tgz", + "integrity": "sha512-Z274K+e8goZK8QJxmbRPhl89HPO1K+ORFtm6rySPhFKfKc5GHhqdzD0SGhSWHkzoXasqJuItdhorSvY7/Cgflw==", + "requires": { + "bluebird": "^3.7.2" + } + }, "unc-path-regex": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/unc-path-regex/-/unc-path-regex-0.1.2.tgz", @@ -7551,6 +8002,11 @@ "spdx-expression-parse": "^3.0.0" } }, + "validator": { + "version": "10.11.0", + "resolved": "https://registry.npmjs.org/validator/-/validator-10.11.0.tgz", + "integrity": "sha512-X/p3UZerAIsbBfN/IwahhYaBbY68EN/UQBWHtsbXGT5bfrH/p4NQzUCG1kF/rtKaNpnJ7jAu6NGTdSNtyNIXMw==" + }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -7663,6 +8119,24 @@ "triple-beam": "^1.2.0" } }, + "winston-transport-sequelize": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/winston-transport-sequelize/-/winston-transport-sequelize-4.0.3.tgz", + "integrity": "sha512-tox3LV0ZzMalgDhuNHlMAqCzjyAxD3ymPmM/0KFTbpy0p6hemSiXlTeeglt7rvaB1LHSMqstYUk5NZbmlGJstQ==", + "requires": { + "sequelize": "^5.21.5", + "winston": "^3.0.0", + "winston-transport": "^4.2.0" + } + }, + "wkx": { + "version": "0.4.8", + "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.8.tgz", + "integrity": "sha512-ikPXMM9IR/gy/LwiOSqWlSL3X/J5uk9EO2hHNRXS41eTLXaUFEVw9fn/593jW/tE5tedNg8YjT5HkCa4FqQZyQ==", + "requires": { + "@types/node": "*" + } + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", diff --git a/package.json b/package.json index b55a01692..5dffd7952 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,6 @@ "async": "^3.1.0", "basic-auth": "^2.0.1", "better-queue": "^3.8.10", - "better-sqlite3": "^5.4.2", "binance-api-node": "^0.9.10", "bitfinex-api-node": "^3.0.2", "bitmex-realtime-api": "^0.4.0", @@ -29,6 +28,7 @@ "commander": "^4.1.0", "compression": "^1.7.4", "cookie-parser": "^1.4.4", + "datatables.net-bs4": "^1.10.20", "express": "^4.16.4", "http": "0.0.0", "lodash": "^4.17.11", @@ -43,12 +43,17 @@ "queue-promise": "^1.3.0", "request": "^2.88.0", "sendmail": "^1.4.1", + "sequelize": "^5.21.7", + "sequelize-cli": "^5.5.1", + "sequelize-datatable": "^2.0.0", + "sqlite3": "^4.2.0", "talib": "^1.0.5", "technicalindicators": "^3.0.0", "telegraf": "^3.35.0", "tulind": "^0.8.15", "twig": "^1.15.0", "winston": "^3.2.1", + "winston-transport-sequelize": "^4.0.3", "ws": "^7.0.0", "zero-fill": "^2.2.3" }, diff --git a/src/modules/listener/ticker_database_listener.js b/src/modules/listener/ticker_database_listener.js index 61ce002d3..4c3c35f05 100644 --- a/src/modules/listener/ticker_database_listener.js +++ b/src/modules/listener/ticker_database_listener.js @@ -2,22 +2,11 @@ const _ = require('lodash'); module.exports = class TickerDatabaseListener { constructor(tickerRepository) { - this.trottle = {}; - - setInterval(async () => { - const tickers = Object.values(this.trottle); - this.trottle = {}; - - if (tickers.length > 0) { - for (const chunk of _.chunk(tickers, 100)) { - await tickerRepository.insertTickers(chunk); - } - } - }, 1000 * 15); + this.tickerRepository = tickerRepository; } onTicker(tickerEvent) { const { ticker } = tickerEvent; - this.trottle[ticker.symbol + ticker.exchange] = ticker; + this.tickerRepository.insertTickers([ticker]); } }; diff --git a/src/modules/repository/candlestick_repository.js b/src/modules/repository/candlestick_repository.js index 99658cf72..f4d3830ab 100644 --- a/src/modules/repository/candlestick_repository.js +++ b/src/modules/repository/candlestick_repository.js @@ -1,89 +1,109 @@ -const Candlestick = require('../../dict/candlestick'); +const { Op } = require('sequelize'); -module.exports = class CandlestickRepository { - constructor(db) { - this.db = db; - } +module.exports = function(sequelize, DataTypes) { + const CandlestickRepository = sequelize.define( + 'Candlestick', + { + exchange: { + type: DataTypes.STRING(255), + primaryKey: true + }, + symbol: { + type: DataTypes.STRING(255), + primaryKey: true + }, + period: { + type: DataTypes.STRING(255), + primaryKey: true, + validate: { + isIn: [['m', 'h', 'd', 'y']] + } + }, + time: { + type: DataTypes.INTEGER, + primaryKey: true, + validate: { + min: 631148400 + } + }, + open: { + type: DataTypes.REAL, + allowNull: true + }, + high: { + type: DataTypes.REAL, + allowNull: true + }, + low: { + type: DataTypes.REAL, + allowNull: true + }, + close: { + type: DataTypes.REAL, + allowNull: true + }, + volume: { + type: DataTypes.REAL, + allowNull: true + } + }, + { + tableName: 'candlesticks' + } + ); - getLookbacksForPair(exchange, symbol, period, limit = 750) { - return new Promise(resolve => { - const stmt = this.db.prepare( - `SELECT * from candlesticks where exchange = ? AND symbol = ? and period = ? order by time DESC LIMIT ${limit}` - ); + CandlestickRepository.getLookbacksForPair = async (exchange, symbol, period, limit = 750) => { + return CandlestickRepository.getCandlesInWindow(exchange, symbol, period, undefined, undefined, limit); + }; - const result = stmt.all([exchange, symbol, period]).map(row => { - return new Candlestick(row.time, row.open, row.high, row.low, row.close, row.volume); - }); + CandlestickRepository.getLookbacksSince = async (exchange, symbol, period, start) => { + return CandlestickRepository.getCandlesInWindow(exchange, symbol, period, start); + }; - resolve(result); - }); - } - - getLookbacksSince(exchange, symbol, period, start) { - return new Promise(resolve => { - const stmt = this.db.prepare( - 'SELECT * from candlesticks where exchange = ? AND symbol = ? and period = ? and time > ? order by time DESC' - ); - - const result = stmt.all([exchange, symbol, period, start]).map(row => { - return new Candlestick(row.time, row.open, row.high, row.low, row.close, row.volume); - }); - - resolve(result); - }); - } - - getCandlesInWindow(exchange, symbol, period, start, end) { - return new Promise(resolve => { - const stmt = this.db.prepare( - 'SELECT * from candlesticks where exchange = ? AND symbol = ? and period = ? and time > ? and time < ? order by time DESC LIMIT 1000' - ); - - const result = stmt - .all([exchange, symbol, period, Math.round(start.getTime() / 1000), Math.round(end.getTime() / 1000)]) - .map(row => { + CandlestickRepository.getCandlesInWindow = async (exchange, symbol, period, start, end, limit = 1000) => { + const whereCondition = { + exchange: exchange, + symbol: symbol, + period: period + }; + const timeConditions = { + ...(typeof(start) !== 'undefined') && {[Op.gt]: Math.round(start.getTime() / 1000)}, + ...(typeof(end) !== 'undefined') && {[Op.lt]: Math.round(end.getTime() / 1000)} + }; + if (Object.keys(timeConditions).length !== 0) { + whereCondition.time = timeConditions; + }; + return CandlestickRepository.findAll({ + // attributes: ['time', 'open', 'high', 'low', 'close', 'volume'], + where: whereCondition, + order: [['time', 'DESC']], + limit: limit + // raw : true + }); + /* .then(candles => { + return candles.map(row => { return new Candlestick(row.time, row.open, row.high, row.low, row.close, row.volume); }); + }) + .catch(err => { + console.log(`Error: ${err}`); + }); */ + }; - resolve(result); + CandlestickRepository.getExchangePairs = async () => { + return CandlestickRepository.findAll({ + group: ['exchange', 'symbol'], + attributes: ['exchange', 'symbol'], + order: ['exchange', 'symbol'], + raw : true }); - } + }; - getExchangePairs() { - return new Promise(resolve => { - const stmt = this.db.prepare( - 'select exchange, symbol from candlesticks group by exchange, symbol order by exchange, symbol' - ); - resolve(stmt.all()); + CandlestickRepository.insertCandles = (exchangeCandlesticks) => { + return CandlestickRepository.bulkCreate(exchangeCandlesticks, { + updateOnDuplicate: ['open', 'high', 'low', 'close', 'volume'] }); - } - - insertCandles(exchangeCandlesticks) { - return new Promise(resolve => { - const upsert = this.db.prepare( - 'INSERT INTO candlesticks(exchange, symbol, period, time, open, high, low, close, volume) VALUES ($exchange, $symbol, $period, $time, $open, $high, $low, $close, $volume) ' + - 'ON CONFLICT(exchange, symbol, period, time) DO UPDATE SET open=$open, high=$high, low=$low, close=$close, volume=$volume' - ); - - this.db.transaction(() => { - exchangeCandlesticks.forEach(exchangeCandlestick => { - const parameters = { - exchange: exchangeCandlestick.exchange, - symbol: exchangeCandlestick.symbol, - period: exchangeCandlestick.period, - time: exchangeCandlestick.time, - open: exchangeCandlestick.open, - high: exchangeCandlestick.high, - low: exchangeCandlestick.low, - close: exchangeCandlestick.close, - volume: exchangeCandlestick.volume - }; + }; - upsert.run(parameters); - }); - })(); - - resolve(); - }); - } + return CandlestickRepository; }; diff --git a/src/modules/repository/logs_repository.js b/src/modules/repository/logs_repository.js deleted file mode 100644 index 00a1f0941..000000000 --- a/src/modules/repository/logs_repository.js +++ /dev/null @@ -1,54 +0,0 @@ -const moment = require('moment'); - -module.exports = class LogsRepository { - constructor(db) { - this.db = db; - } - - getLatestLogs(filters = { logExcludeLevels: ['debug'], logTxtFilter: '' }, limit = 200) { - return new Promise(resolve => { - const parameters = {}; - let sql = `SELECT * from logs WHERE 1=1`; - - if (filters.logExcludeLevels.length > 0) { - sql += ` AND level NOT IN (${filters.logExcludeLevels - .map((_exclude, index) => `$level_${index}`) - .join(', ')}) `; - - filters.logExcludeLevels.forEach((exclude, index) => { - parameters[`level_${index}`] = exclude; - }); - } - - if (filters.logTxtFilter !== '') { - sql += ` AND message LIKE '%${filters.logTxtFilter}%'`; - } - - sql += ` order by created_at DESC LIMIT ${limit}`; - - const stmt = this.db.prepare(sql); - resolve(stmt.all(parameters)); - }); - } - - getLevels() { - return new Promise(resolve => { - const stmt = this.db.prepare('SELECT level from logs GROUP BY level'); - resolve(stmt.all().map(r => r.level)); - }); - } - - cleanOldLogEntries(days = 7) { - return new Promise(resolve => { - const stmt = this.db.prepare('DELETE FROM logs WHERE created_at < $created_at'); - - stmt.run({ - created_at: moment() - .subtract(days, 'days') - .unix() - }); - - resolve(); - }); - } -}; diff --git a/src/modules/repository/signal_repository.js b/src/modules/repository/signal_repository.js index fae9dcc4f..1f6995ca0 100644 --- a/src/modules/repository/signal_repository.js +++ b/src/modules/repository/signal_repository.js @@ -1,27 +1,58 @@ -module.exports = class SignalRepository { - constructor(db) { - this.db = db; - } +const { Op } = require('sequelize'); - getSignals(since) { - return new Promise(resolve => { - const stmt = this.db.prepare('SELECT * from signals where income_at > ? order by income_at DESC LIMIT 100'); - resolve(stmt.all(since)); +module.exports = function(sequelize, DataTypes) { + const SignalRepository = sequelize.define( + 'Signal', + { + exchange: { + type: DataTypes.STRING(255), + allowNull: true + }, + symbol: { + type: DataTypes.STRING(255), + allowNull: true + }, + ask: { + type: DataTypes.REAL, + allowNull: true + }, + bid: { + type: DataTypes.REAL, + allowNull: true + }, + options: { + type: DataTypes.TEXT, + allowNull: true + }, + side: { + type: DataTypes.STRING(50), + allowNull: true + }, + strategy: { + type: DataTypes.STRING(50), + allowNull: true + }, + state: { + type: DataTypes.STRING(50), + allowNull: true + } + }, + { + tableName: 'signals' + } + ); + + SignalRepository.getSignals = async since => { + return SignalRepository.findAll({ + where: { updatedAt: { [Op.gt]: since } }, + order: [['updatedAt', 'DESC']], + limit: 1000 + // raw : true }); - } + }; - insertSignal(exchange, symbol, options, side, strategy) { - const stmt = this.db.prepare( - 'INSERT INTO signals(exchange, symbol, options, side, strategy, income_at) VALUES ($exchange, $symbol, $options, $side, $strategy, $income_at)' - ); + SignalRepository.insertSignal = async (exchange, symbol, options, side, strategy) => + SignalRepository.create(exchange, symbol, options, side, strategy); - stmt.run({ - exchange: exchange, - symbol: symbol, - options: JSON.stringify(options || {}), - side: side, - strategy: strategy, - income_at: Math.floor(Date.now() / 1000) - }); - } + return SignalRepository; }; diff --git a/src/modules/repository/ticker_log_repository.js b/src/modules/repository/ticker_log_repository.js deleted file mode 100644 index 17f4464ea..000000000 --- a/src/modules/repository/ticker_log_repository.js +++ /dev/null @@ -1,21 +0,0 @@ -const moment = require('moment'); - -module.exports = class TickerLogRepository { - constructor(db) { - this.db = db; - } - - cleanOldLogEntries(days = 14) { - return new Promise(resolve => { - const stmt = this.db.prepare('DELETE FROM ticker_log WHERE income_at < $income_at'); - - stmt.run({ - income_at: moment() - .subtract(days, 'days') - .unix() - }); - - resolve(); - }); - } -}; diff --git a/src/modules/repository/ticker_repository.js b/src/modules/repository/ticker_repository.js index e4b91db34..cc5de9820 100644 --- a/src/modules/repository/ticker_repository.js +++ b/src/modules/repository/ticker_repository.js @@ -1,31 +1,36 @@ -module.exports = class TickerRepository { - constructor(db, logger) { - this.db = db; - this.logger = logger; - } +module.exports = function(sequelize, DataTypes) { + const TickerRepository = sequelize.define( + 'Ticker', + { + exchange: { + type: DataTypes.STRING(255), + allowNull: true, + primaryKey: true + }, + symbol: { + type: DataTypes.STRING(255), + allowNull: true, + primaryKey: true + }, + ask: { + type: DataTypes.REAL, + allowNull: true + }, + bid: { + type: DataTypes.REAL, + allowNull: true + } + }, + { + tableName: 'tickers' + } + ); - insertTickers(tickers) { - return new Promise(resolve => { - const upsert = this.db.prepare( - 'INSERT INTO ticker(exchange, symbol, ask, bid, updated_at) VALUES ($exchange, $symbol, $ask, $bid, $updated_at) ' + - 'ON CONFLICT(exchange, symbol) DO UPDATE SET ask=$ask, bid=$bid, updated_at=$updated_at' - ); - - this.db.transaction(() => { - tickers.forEach(ticker => { - const parameters = { - exchange: ticker.exchange, - symbol: ticker.symbol, - ask: ticker.ask, - bid: ticker.bid, - updated_at: new Date().getTime() - }; - - upsert.run(parameters); - }); - })(); - - resolve(); + TickerRepository.insertTickers = function(tickers) { + return TickerRepository.bulkCreate(tickers, { + updateOnDuplicate: ['ask', 'bid'] }); - } + }; + + return TickerRepository; }; diff --git a/src/modules/services.js b/src/modules/services.js index 050316e8a..066586ef1 100644 --- a/src/modules/services.js +++ b/src/modules/services.js @@ -1,10 +1,14 @@ -const fs = require('fs'); +const Fs = require('fs'); +const Path = require('path'); const events = require('events'); const { createLogger, transports, format } = require('winston'); +const WinstonTransportSequelize = require('winston-transport-sequelize'); const _ = require('lodash'); -const Sqlite = require('better-sqlite3'); +const Sqlite = require('sqlite3').verbose(); +const Sequelize = require('sequelize'); +const Datatable = require('sequelize-datatable'); const Notify = require('../notify/notify'); const Slack = require('../notify/slack'); const Mail = require('../notify/mail'); @@ -23,7 +27,7 @@ const SignalLogger = require('../modules/signal/signal_logger'); const SignalHttp = require('../modules/signal/signal_http'); const SignalRepository = require('../modules/repository/signal_repository'); -const CandlestickRepository = require('../modules/repository/candlestick_repository'); +const CandlestickRepository = require('../modules/repository/candlestick_repository').default; const StrategyManager = require('./strategy/strategy_manager'); const ExchangeManager = require('./exchange/exchange_manager'); @@ -42,11 +46,7 @@ const PairStateExecution = require('../modules/pairs/pair_state_execution'); const PairConfig = require('../modules/pairs/pair_config'); const SystemUtil = require('../modules/system/system_util'); const TechnicalAnalysisValidator = require('../utils/technical_analysis_validator'); -const WinstonSqliteTransport = require('../utils/winston_sqlite_transport'); const LogsHttp = require('./system/logs_http'); -const LogsRepository = require('../modules/repository/logs_repository'); -const TickerLogRepository = require('../modules/repository/ticker_log_repository'); -const TickerRepository = require('../modules/repository/ticker_repository'); const CandlestickResample = require('../modules/system/candlestick_resample'); const RequestClient = require('../utils/request_client'); const Queue = require('../utils/queue'); @@ -106,7 +106,6 @@ let systemUtil; let technicalAnalysisValidator; let logsHttp; let logsRepository; -let tickerLogRepository; let candlestickResample; let exchanges; let requestClient; @@ -135,12 +134,18 @@ module.exports = { } try { - config = JSON.parse(fs.readFileSync(`${parameters.projectDir}/conf.json`, 'utf8')); + config = JSON.parse(Fs.readFileSync(Path.join(parameters.projectDir, 'conf.json'), 'utf8')); } catch (e) { throw new Error(`Invalid conf.json file. Please check: ${String(e)}`); } this.getDatabase(); + this.getLogger(); + await db.sequelize.sync({ force: true }); + + /* const zz = await db.Candlestick.getLookbacksForPair('bitfinex', 'BTCUSD', '1h'); + const yy = await db.Candlestick.getExchangePairs(); + console.log('zz'); */ }, getDatabase: () => { @@ -148,13 +153,25 @@ module.exports = { return db; } - const myDb = Sqlite('bot.db'); - myDb.pragma('journal_mode = WAL'); + const dbSettings = { + dialect: _.get(config, 'production.dialect'), + storage: _.get(config, 'production.storage') + }; + + const sequelize = new Sequelize(dbSettings.database, dbSettings.user, dbSettings.password, dbSettings); + + db = {}; + Fs.readdirSync(Path.join(parameters.projectDir, 'src', 'modules', 'repository')) + .filter(file => file.indexOf('.') !== 0 && file !== 'index.js') + .forEach(file => { + const model = sequelize.import(Path.join(parameters.projectDir, 'src', 'modules', 'repository', file)); + db[model.name] = model; + }); - myDb.pragma('SYNCHRONOUS = 1;'); - myDb.pragma('LOCKING_MODE = EXCLUSIVE;'); + db.sequelize = sequelize; + db.Sequelize = Sequelize; - return (db = myDb); + return db; }, getTa: function() { @@ -202,7 +219,8 @@ module.exports = { return createOrderListener; } - return (createOrderListener = new CreateOrderListener(this.getExchangeManager(), this.getLogger())); + createOrderListener = new CreateOrderListener(this.getExchangeManager(), this.getLogger()); + return createOrderListener; }, getTickListener: function() { @@ -210,7 +228,7 @@ module.exports = { return tickListener; } - return (tickListener = new TickListener( + tickListener = new TickListener( this.getTickers(), this.getInstances(), this.getNotifier(), @@ -220,7 +238,8 @@ module.exports = { this.getPairStateManager(), this.getLogger(), this.getSystemUtil() - )); + ); + return tickListener; }, getExchangeOrderWatchdogListener: function() { @@ -228,7 +247,7 @@ module.exports = { return exchangeOrderWatchdogListener; } - return (exchangeOrderWatchdogListener = new ExchangeOrderWatchdogListener( + exchangeOrderWatchdogListener = new ExchangeOrderWatchdogListener( this.getExchangeManager(), this.getInstances(), this.getStopLossCalculator(), @@ -237,7 +256,8 @@ module.exports = { this.getPairStateManager(), this.getLogger(), this.getTickers() - )); + ); + return exchangeOrderWatchdogListener; }, getTickerDatabaseListener: function() { @@ -245,7 +265,8 @@ module.exports = { return tickerDatabaseListener; } - return (tickerDatabaseListener = new TickerDatabaseListener(this.getTickerRepository())); + tickerDatabaseListener = new TickerDatabaseListener(this.getTickerRepository()); + return tickerDatabaseListener; }, getSignalLogger: function() { @@ -253,7 +274,8 @@ module.exports = { return signalLogger; } - return (signalLogger = new SignalLogger(this.getSignalRepository())); + signalLogger = new SignalLogger(this.getSignalRepository()); + return signalLogger; }, getSignalHttp: function() { @@ -261,7 +283,8 @@ module.exports = { return signalHttp; } - return (signalHttp = new SignalHttp(this.getSignalRepository())); + signalHttp = new SignalHttp(this.getSignalRepository()); + return signalHttp; }, getSignalRepository: function() { @@ -269,7 +292,8 @@ module.exports = { return signalRepository; } - return (signalRepository = new SignalRepository(this.getDatabase())); + signalRepository = this.getDatabase().Signal; + return signalRepository; }, getCandlestickRepository: function() { @@ -277,7 +301,8 @@ module.exports = { return candlestickRepository; } - return (candlestickRepository = new CandlestickRepository(this.getDatabase())); + candlestickRepository = this.getDatabase().Candlestick; + return candlestickRepository; }, getEventEmitter: function() { @@ -293,7 +318,15 @@ module.exports = { return logger; } - return (logger = createLogger({ + const options = { + sequelize: this.getDatabase().sequelize, // sequelize instance [required] + tableName: 'log', // default name + meta: { project: 'crypto-trading-bot' }, // meta object defaults + fields: { meta: Sequelize.JSONB }, // merge model fields + modelOptions: { timestamps: false } // merge model options + }; + + logger = createLogger({ format: format.combine(format.timestamp(), format.json()), transports: [ new transports.File({ @@ -303,13 +336,10 @@ module.exports = { new transports.Console({ level: 'error' }), - new WinstonSqliteTransport({ - level: 'debug', - database_connection: this.getDatabase(), - table: 'logs' - }) + new WinstonTransportSequelize(options) ] - })); + }); + return logger; }, getNotifier: function() { @@ -477,7 +507,7 @@ module.exports = { return logsRepository; } - return (logsRepository = new LogsRepository(this.getDatabase())); + return (logsRepository = this.getDatabase().Log); }, getLogsHttp: function() { @@ -488,20 +518,12 @@ module.exports = { return (logsHttp = new LogsHttp(this.getLogsRepository())); }, - getTickerLogRepository: function() { - if (tickerLogRepository) { - return tickerLogRepository; - } - - return (tickerLogRepository = new TickerLogRepository(this.getDatabase())); - }, - getTickerRepository: function() { if (tickerRepository) { return tickerRepository; } - return (tickerRepository = new TickerRepository(this.getDatabase(), this.getLogger())); + return (tickerRepository = this.getDatabase().Ticker); }, getCandlestickResample: function() { @@ -646,7 +668,6 @@ module.exports = { this.getPairStateExecution(), this.getSystemUtil(), this.getLogsRepository(), - this.getTickerLogRepository(), this.getExchangePositionWatcher() ); }, diff --git a/src/modules/trade.js b/src/modules/trade.js index 9cc453bff..ab57fb074 100644 --- a/src/modules/trade.js +++ b/src/modules/trade.js @@ -18,7 +18,6 @@ module.exports = class Trade { pairStateExecution, systemUtil, logsRepository, - tickerLogRepository, exchangePositionWatcher ) { this.eventEmitter = eventEmitter; @@ -34,7 +33,6 @@ module.exports = class Trade { this.pairStateExecution = pairStateExecution; this.systemUtil = systemUtil; this.logsRepository = logsRepository; - this.tickerLogRepository = tickerLogRepository; this.exchangePositionWatcher = exchangePositionWatcher; } @@ -96,9 +94,7 @@ module.exports = class Trade { // cronjob like tasks setInterval(async () => { - await me.logsRepository.cleanOldLogEntries(); - await me.tickerLogRepository.cleanOldLogEntries(); - + // await me.logsRepository.cleanOldLogEntries(); me.logger.debug('Logs: Cleanup old entries'); }, 86455000); diff --git a/src/utils/winston_sqlite_transport.js b/src/utils/winston_sqlite_transport.js deleted file mode 100644 index 292d85627..000000000 --- a/src/utils/winston_sqlite_transport.js +++ /dev/null @@ -1,48 +0,0 @@ -const Transport = require('winston-transport'); - -module.exports = class WinstonSqliteTransport extends Transport { - constructor(opts) { - super(opts); - - if (!opts.database_connection) { - throw new Error('database_connection is needed'); - } - - if (!opts.table) { - throw new Error('table is needed'); - } - - this.db = opts.database_connection; - this.table = opts.table; - } - - log(info, callback) { - setImmediate(() => { - this.emit('logged', info); - }); - - const parameters = { - uuid: WinstonSqliteTransport.createUUID(), - level: info.level, - message: info.message, - created_at: Math.floor(Date.now() / 1000) - }; - - this.db - .prepare( - `INSERT INTO ${this.table}(uuid, level, message, created_at) VALUES ($uuid, $level, $message, $created_at)` - ) - .run(parameters); - - callback(); - } - - static createUUID() { - let dt = new Date().getTime(); - return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { - const r = (dt + Math.random() * 16) % 16 | 0; - dt = Math.floor(dt / 16); - return (c == 'x' ? r : (r & 0x3) | 0x8).toString(16); - }); - } -}; From 25dcae1585c3e3094fb29fea8a85c0cdfa5f3f27 Mon Sep 17 00:00:00 2001 From: egisz Date: Mon, 11 May 2020 18:12:27 +0300 Subject: [PATCH 04/25] fix date filters, make same based on timestamp --- src/modules/repository/candlestick_repository.js | 4 ++-- src/modules/system/candle_export_http.js | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/modules/repository/candlestick_repository.js b/src/modules/repository/candlestick_repository.js index f4d3830ab..abe1c956a 100644 --- a/src/modules/repository/candlestick_repository.js +++ b/src/modules/repository/candlestick_repository.js @@ -67,8 +67,8 @@ module.exports = function(sequelize, DataTypes) { period: period }; const timeConditions = { - ...(typeof(start) !== 'undefined') && {[Op.gt]: Math.round(start.getTime() / 1000)}, - ...(typeof(end) !== 'undefined') && {[Op.lt]: Math.round(end.getTime() / 1000)} + ...(typeof(start) !== 'undefined') && {[Op.gt]: start}, + ...(typeof(end) !== 'undefined') && {[Op.lt]: end} }; if (Object.keys(timeConditions).length !== 0) { whereCondition.time = timeConditions; diff --git a/src/modules/system/candle_export_http.js b/src/modules/system/candle_export_http.js index 792888a91..cd16801a7 100644 --- a/src/modules/system/candle_export_http.js +++ b/src/modules/system/candle_export_http.js @@ -4,7 +4,13 @@ module.exports = class CandleExportHttp { } async getCandles(exchange, symbol, period, start, end) { - return this.candlestickRepository.getCandlesInWindow(exchange, symbol, period, start, end); + return this.candlestickRepository.getCandlesInWindow( + exchange, + symbol, + period, + start.getTime() / 1000, + end.getTime() / 1000 + ); } async getPairs() { From 40fca20ec9b06ba95745841ea97d61678836741e Mon Sep 17 00:00:00 2001 From: egisz Date: Mon, 11 May 2020 22:38:59 +0300 Subject: [PATCH 05/25] limit columns in query result --- src/modules/repository/candlestick_repository.js | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/modules/repository/candlestick_repository.js b/src/modules/repository/candlestick_repository.js index abe1c956a..86699da09 100644 --- a/src/modules/repository/candlestick_repository.js +++ b/src/modules/repository/candlestick_repository.js @@ -74,20 +74,12 @@ module.exports = function(sequelize, DataTypes) { whereCondition.time = timeConditions; }; return CandlestickRepository.findAll({ - // attributes: ['time', 'open', 'high', 'low', 'close', 'volume'], + attributes: ['time', 'open', 'high', 'low', 'close', 'volume'], where: whereCondition, order: [['time', 'DESC']], - limit: limit - // raw : true + limit: limit, + raw : true }); - /* .then(candles => { - return candles.map(row => { - return new Candlestick(row.time, row.open, row.high, row.low, row.close, row.volume); - }); - }) - .catch(err => { - console.log(`Error: ${err}`); - }); */ }; CandlestickRepository.getExchangePairs = async () => { From fe2cbcec70cf4ff20560c74da5560f869157702f Mon Sep 17 00:00:00 2001 From: egisz Date: Tue, 12 May 2020 11:58:45 +0300 Subject: [PATCH 06/25] fix winston sequelize logging paramaters --- src/modules/services.js | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/modules/services.js b/src/modules/services.js index 6b6611999..23b5d8984 100644 --- a/src/modules/services.js +++ b/src/modules/services.js @@ -6,9 +6,7 @@ const { createLogger, transports, format } = require('winston'); const WinstonTransportSequelize = require('winston-transport-sequelize'); const _ = require('lodash'); -const Sqlite = require('sqlite3').verbose(); const Sequelize = require('sequelize'); -const Datatable = require('sequelize-datatable'); const Notify = require('../notify/notify'); const Slack = require('../notify/slack'); const Mail = require('../notify/mail'); @@ -26,8 +24,6 @@ const ExchangePositionWatcher = require('../modules/exchange/exchange_position_w const SignalLogger = require('../modules/signal/signal_logger'); const SignalHttp = require('../modules/signal/signal_http'); -const SignalRepository = require('../modules/repository/signal_repository'); -const CandlestickRepository = require('../modules/repository/candlestick_repository').default; const StrategyManager = require('./strategy/strategy_manager'); const ExchangeManager = require('./exchange/exchange_manager'); @@ -142,10 +138,6 @@ module.exports = { this.getDatabase(); this.getLogger(); await db.sequelize.sync({ force: true }); - - /* const zz = await db.Candlestick.getLookbacksForPair('bitfinex', 'BTCUSD', '1h'); - const yy = await db.Candlestick.getExchangePairs(); - console.log('zz'); */ }, getDatabase: () => { @@ -320,12 +312,11 @@ module.exports = { const options = { sequelize: this.getDatabase().sequelize, // sequelize instance [required] - tableName: 'log', // default name - meta: { project: 'crypto-trading-bot' }, // meta object defaults - fields: { meta: Sequelize.JSONB }, // merge model fields - modelOptions: { timestamps: false } // merge model options + tableName: 'logs', // default name + level: 'debug' }; + const winstonTransportSequelize = new WinstonTransportSequelize(options); logger = createLogger({ format: format.combine(format.timestamp(), format.json()), transports: [ @@ -336,9 +327,10 @@ module.exports = { new transports.Console({ level: 'error' }), - new WinstonTransportSequelize(options) + winstonTransportSequelize ] }); + this.getDatabase().Log = winstonTransportSequelize.model; return logger; }, From 921dbd90cfaa4fbf48d48c0886631d2413622eb6 Mon Sep 17 00:00:00 2001 From: egisz Date: Wed, 20 May 2020 09:18:29 +0300 Subject: [PATCH 07/25] fix log page filters, source bootstrap and datatables libraries from npm. --- README.md | 4 +- package-lock.json | 144 +++++++++----------------------- package.json | 6 +- src/modules/http.js | 14 +++- src/modules/services.js | 2 +- src/modules/system/logs_http.js | 59 +++---------- templates/logs.html.twig | 19 ++--- web/static/js/logs.js | 54 ++++++++---- 8 files changed, 115 insertions(+), 187 deletions(-) diff --git a/README.md b/README.md index 4f97353bc..7568ed414 100644 --- a/README.md +++ b/README.md @@ -39,12 +39,14 @@ TODOS: * node.js * sqlite3 + * [sequelize ORM](https://sequelize.org/) * [technicalindicators](https://github.com/anandanand84/technicalindicators) * [tulipindicators - tulind](https://tulipindicators.org/list) * [TA-Lib](https://mrjbq7.github.io/ta-lib/) - * twig + * [twig](https://www.npmjs.com/package/twig) * express * Bootstrap v4 + * [datatables](https://datatables.net/) * Tradingview widgets ## How to use diff --git a/package-lock.json b/package-lock.json index bbf74b431..ce9f1f085 100644 --- a/package-lock.json +++ b/package-lock.json @@ -234,11 +234,6 @@ "form-data": "*" } }, - "@types/geojson": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-1.0.6.tgz", - "integrity": "sha512-Xqg/lIZMrUd0VRmSRbCAewtwGZiAk3mEUDvV4op1tGl+LvyPcb/MIOSxTl9z+9+J+R4/vpjiCAT4xeKzH9ji1w==" - }, "@types/node": { "version": "12.12.17", "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.17.tgz", @@ -1684,6 +1679,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/cli-color/-/cli-color-1.4.0.tgz", "integrity": "sha512-xu6RvQqqrWEo6MPR1eixqGPywhYBHRs653F9jfXB2Hx4jdM/3WxiNE1vppRmxtMIfl16SFYTpYlrnqH/HsK/2w==", + "dev": true, "requires": { "ansi-regex": "^2.1.1", "d": "1", @@ -1905,6 +1901,7 @@ "version": "1.1.12", "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.12.tgz", "integrity": "sha512-a1eOIcu8+7lUInge4Rpf/n4Krkf3Dd9lqhljRzII1/Zno/kRtUWnznPO3jOKBmTEktkt3fkxisUcivoj0ebzoA==", + "dev": true, "requires": { "ini": "^1.3.4", "proto-list": "~1.2.1" @@ -2057,6 +2054,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", + "dev": true, "requires": { "es5-ext": "^0.10.50", "type": "^1.0.1" @@ -2226,6 +2224,7 @@ "version": "0.15.3", "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz", "integrity": "sha512-M9wIMFx96vq0R4F+gRpY3o2exzb8hEj/n9S8unZtHSvYjibBp/iMufSzvmOcV/laG0ZtuTVGtiJggPOSW2r93g==", + "dev": true, "requires": { "commander": "^2.19.0", "lru-cache": "^4.1.5", @@ -2236,7 +2235,8 @@ "commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true } } }, @@ -2330,6 +2330,7 @@ "version": "0.10.53", "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz", "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==", + "dev": true, "requires": { "es6-iterator": "~2.0.3", "es6-symbol": "~3.1.3", @@ -2340,6 +2341,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=", + "dev": true, "requires": { "d": "1", "es5-ext": "^0.10.35", @@ -2355,6 +2357,7 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", + "dev": true, "requires": { "d": "^1.0.1", "ext": "^1.1.2" @@ -2364,6 +2367,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.3.tgz", "integrity": "sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==", + "dev": true, "requires": { "d": "1", "es5-ext": "^0.10.46", @@ -3395,6 +3399,7 @@ "version": "0.3.5", "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", "integrity": "sha1-34xp7vFkeSPHFXuc6DhAYQsCzDk=", + "dev": true, "requires": { "d": "1", "es5-ext": "~0.10.14" @@ -3516,6 +3521,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", "integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==", + "dev": true, "requires": { "type": "^2.0.0" }, @@ -3523,7 +3529,8 @@ "type": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/type/-/type-2.0.0.tgz", - "integrity": "sha512-KBt58xCHry4Cejnc2ISQAF7QY+ORngsWfxezO68+12hKV6lQY8P/psIkcbjeHWn7MqcgciWJyCCevFMJdIXpow==" + "integrity": "sha512-KBt58xCHry4Cejnc2ISQAF7QY+ORngsWfxezO68+12hKV6lQY8P/psIkcbjeHWn7MqcgciWJyCCevFMJdIXpow==", + "dev": true } } }, @@ -3922,11 +3929,6 @@ } } }, - "generic-pool": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.5.0.tgz", - "integrity": "sha512-dEkxmX+egB2o4NR80c/q+xzLLzLX+k68/K8xv81XprD+Sk7ZtP14VugeCz+fUwv5FzpWq40pPtAkzPRqT8ka9w==" - }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -4550,7 +4552,8 @@ "is-promise": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", - "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=" + "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=", + "dev": true }, "is-redirect": { "version": "1.0.0", @@ -4666,6 +4669,7 @@ "version": "1.11.0", "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.11.0.tgz", "integrity": "sha512-a26B+Cx7USQGSWnz9YxgJNMmML/QG2nqIaL7VVYPCXbqiKz8PN0waSNvroMtvAK6tY7g/wPdNWGEP+JTNIBr6A==", + "dev": true, "requires": { "config-chain": "^1.1.12", "editorconfig": "^0.15.3", @@ -4677,12 +4681,14 @@ "mkdirp": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true }, "nopt": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", + "dev": true, "requires": { "abbrev": "1", "osenv": "^0.1.4" @@ -4987,6 +4993,7 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/lru-queue/-/lru-queue-0.1.0.tgz", "integrity": "sha1-Jzi9nw089PhEkMVzbEhpmsYyzaM=", + "dev": true, "requires": { "es5-ext": "~0.10.2" } @@ -5063,6 +5070,7 @@ "version": "0.4.14", "resolved": "https://registry.npmjs.org/memoizee/-/memoizee-0.4.14.tgz", "integrity": "sha512-/SWFvWegAIYAO4NQMpcX+gcra0yEZu4OntmUdrBaWrJncxOqAziGFlHxc7yjKVK2uu3lpPW27P27wkR82wA8mg==", + "dev": true, "requires": { "d": "1", "es5-ext": "^0.10.45", @@ -5481,7 +5489,8 @@ "next-tick": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", - "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" + "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=", + "dev": true }, "nice-try": { "version": "1.0.5", @@ -5995,7 +6004,8 @@ "path-parse": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==" + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "dev": true }, "path-to-regexp": { "version": "0.1.7", @@ -6164,7 +6174,8 @@ "proto-list": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", - "integrity": "sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=" + "integrity": "sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=", + "dev": true }, "proxy-addr": { "version": "2.0.5", @@ -6510,6 +6521,7 @@ "version": "1.13.1", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.13.1.tgz", "integrity": "sha512-CxqObCX8K8YtAhOBRg+lrcdn+LK+WYOS8tSjqSFbjtrI5PnS63QPhZl4+yKfrU9tdsbMu9Anr/amegT87M9Z6w==", + "dev": true, "requires": { "path-parse": "^1.0.6" } @@ -6711,6 +6723,7 @@ "version": "5.5.1", "resolved": "https://registry.npmjs.org/sequelize-cli/-/sequelize-cli-5.5.1.tgz", "integrity": "sha512-ZM4kUZvY3y14y+Rq3cYxGH7YDJz11jWHcN2p2x7rhAIemouu4CEXr5ebw30lzTBtyXV4j2kTO+nUjZOqzG7k+Q==", + "dev": true, "requires": { "bluebird": "^3.5.3", "cli-color": "^1.4.0", @@ -6722,77 +6735,13 @@ "yargs": "^13.1.0" } }, - "sequelize-datatable": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/sequelize-datatable/-/sequelize-datatable-2.0.0.tgz", - "integrity": "sha512-OAc31wVkbEyo24kJpZ2DP+X4HGAcsGxiGQIvuoqlRHiXYcgPJu+i+sSQlL3Pd43cG3Jr8e3JpId+8OgPtCbLJQ==", + "sequelize-datatables": { + "version": "git+https://git@github.com/egisz/sequelize-datatable-node.git#08124feaf79b7c38b5e39513330d20016c61bb6a", + "from": "git+https://git@github.com/egisz/sequelize-datatable-node.git", "requires": { "bluebird": "^3.5.1", "lodash": "^4.17.10", - "sequelize": "^4.37.10" - }, - "dependencies": { - "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "retry-as-promised": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-2.3.2.tgz", - "integrity": "sha1-zZdO5P2bX+A8vzGHHuSCIcB3N7c=", - "requires": { - "bluebird": "^3.4.6", - "debug": "^2.6.9" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - } - } - }, - "sequelize": { - "version": "4.44.4", - "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-4.44.4.tgz", - "integrity": "sha512-nkHmYkbwQK7uwpgW9VBalCBnQqQ8mslTdgcBthtJLORuPvAYRPlfkXZMVUU9TLLJt9CX+/y0MYg0DpcP6ywsEQ==", - "requires": { - "bluebird": "^3.5.0", - "cls-bluebird": "^2.1.0", - "debug": "^3.1.0", - "depd": "^1.1.0", - "dottie": "^2.0.0", - "generic-pool": "3.5.0", - "inflection": "1.12.0", - "lodash": "^4.17.1", - "moment": "^2.20.0", - "moment-timezone": "^0.5.14", - "retry-as-promised": "^2.3.2", - "semver": "^5.5.0", - "terraformer-wkt-parser": "^1.1.2", - "toposort-class": "^1.0.1", - "uuid": "^3.2.1", - "validator": "^10.4.0", - "wkx": "^0.4.1" - } - } + "sequelize": "^5.21.7" } }, "sequelize-pool": { @@ -6861,7 +6810,8 @@ "sigmund": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz", - "integrity": "sha1-P/IfGYytIXX587eBhT/ZTQ0ZtZA=" + "integrity": "sha1-P/IfGYytIXX587eBhT/ZTQ0ZtZA=", + "dev": true }, "signal-exit": { "version": "3.0.2", @@ -7441,23 +7391,6 @@ "execa": "^0.7.0" } }, - "terraformer": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/terraformer/-/terraformer-1.0.12.tgz", - "integrity": "sha512-MokUp0+MFal4CmJDVL6VAO1bKegeXcBM2RnPVfqcFIp2IIv8EbPAjG0j/vEy/vuKB8NVMMSF2vfpVS/QLe4DBg==", - "requires": { - "@types/geojson": "^7946.0.0 || ^1.0.0" - } - }, - "terraformer-wkt-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/terraformer-wkt-parser/-/terraformer-wkt-parser-1.2.1.tgz", - "integrity": "sha512-+CJyNLWb3lJ9RsZMTM66BY0MT3yIo4l4l22Jd9CrZuwzk54fsu4Sc7zejuS9fCITTuTQy3p06d4MZMVI7v5wSg==", - "requires": { - "@types/geojson": "^1.0.0", - "terraformer": "~1.0.5" - } - }, "text-hex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", @@ -7493,6 +7426,7 @@ "version": "0.1.7", "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.7.tgz", "integrity": "sha512-b85NUNzTSdodShTIbky6ZF02e8STtVVfD+fu4aXXShEELpozH+bCpJLYMPZbsABN2wDH7fJpqIoXxJpzbf0NqQ==", + "dev": true, "requires": { "es5-ext": "~0.10.46", "next-tick": "1" @@ -7768,7 +7702,8 @@ "type": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", - "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" + "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==", + "dev": true }, "type-check": { "version": "0.3.2", @@ -7803,6 +7738,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/umzug/-/umzug-2.3.0.tgz", "integrity": "sha512-Z274K+e8goZK8QJxmbRPhl89HPO1K+ORFtm6rySPhFKfKc5GHhqdzD0SGhSWHkzoXasqJuItdhorSvY7/Cgflw==", + "dev": true, "requires": { "bluebird": "^3.7.2" } diff --git a/package.json b/package.json index 5dffd7952..62d1323f9 100644 --- a/package.json +++ b/package.json @@ -44,8 +44,7 @@ "request": "^2.88.0", "sendmail": "^1.4.1", "sequelize": "^5.21.7", - "sequelize-cli": "^5.5.1", - "sequelize-datatable": "^2.0.0", + "sequelize-datatables": "git+https://git@github.com/egisz/sequelize-datatable-node.git", "sqlite3": "^4.2.0", "talib": "^1.0.5", "technicalindicators": "^3.0.0", @@ -101,6 +100,7 @@ "eslint-plugin-prettier": "^3.1.2", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-react": "^7.18.0", - "prettier": "^1.19.1" + "prettier": "^1.19.1", + "sequelize-cli": "^5.5.1" } } diff --git a/src/modules/http.js b/src/modules/http.js index 1bbcc1220..0cfa1b7eb 100644 --- a/src/modules/http.js +++ b/src/modules/http.js @@ -5,6 +5,7 @@ const auth = require('basic-auth'); const cookieParser = require('cookie-parser'); const crypto = require('crypto'); const moment = require('moment'); +const Path = require('path'); module.exports = class Http { constructor( @@ -80,6 +81,16 @@ module.exports = class Http { app.use(cookieParser()); app.use(compression()); app.use(express.static(`${this.projectDir}/web/static`, { maxAge: 3600000 * 24 })); + app.use('/scripts/moment', express.static(Path.join(this.projectDir, '/node_modules/moment/min'))); + app.use('/scripts/datatables.net', express.static(Path.join(this.projectDir, '/node_modules/datatables.net/js'))); + app.use( + '/scripts/datatables.net-bs4/', + express.static(Path.join(this.projectDir, '/node_modules/datatables.net-bs4/js')) + ); + app.use( + '/css/datatables.net-bs4/', + express.static(Path.join(this.projectDir, '/node_modules/datatables.net-bs4/css')) + ); const username = this.systemUtil.getConfig('webserver.username'); const password = this.systemUtil.getConfig('webserver.password'); @@ -153,8 +164,7 @@ module.exports = class Http { }); app.post('/logsTable', async (req, res) => { - const logs = await this.logsHttp.getLogsData(req, res); - res.json(logs); + res.json(await this.logsHttp.getLogsData(req, res)); }); app.get('/desks/:desk', async (req, res) => { diff --git a/src/modules/services.js b/src/modules/services.js index 23b5d8984..40103a57c 100644 --- a/src/modules/services.js +++ b/src/modules/services.js @@ -137,7 +137,7 @@ module.exports = { this.getDatabase(); this.getLogger(); - await db.sequelize.sync({ force: true }); + await db.sequelize.sync(); }, getDatabase: () => { diff --git a/src/modules/system/logs_http.js b/src/modules/system/logs_http.js index bf51dd3f0..b5a7d0983 100644 --- a/src/modules/system/logs_http.js +++ b/src/modules/system/logs_http.js @@ -1,60 +1,23 @@ +const Datatable = require('sequelize-datatables'); + module.exports = class LogsHttp { constructor(logsRepository) { this.logsRepository = logsRepository; } async getLogsPageVariables(request, response) { - const filters = request.query.filters || - request.cookies.filters || { logExcludeLevels: ['debug'], logTxtFilter: '' }; - - response.cookie('filters', filters, { - maxAge: 60 * 60 * 24 * 30 * 1000 - }); - - // We will highlight profit number in logs output - const logs = await this.logsRepository.getLatestLogs(filters); - logs.forEach(log => { - const profit = log.message.match(/(?<=profit":)(-?\d+.\d+)/); - if (profit) { - [ log.profit ] = profit; - } - }); - return { - logs: logs, - levels: await this.logsRepository.getLevels(), - form: { - filters: filters - } + levels: await this.logsRepository + .findAll({ + attributes: ['level'], + group: ['level'], + raw: true + }) + .map(row => row.level) }; } - async getLogsData2(request, response) { - - // We will highlight profit number in logs output - const logs = await this.logsRepository.getFilteredLogs(request.body); - logs.forEach(log => { - const profit = log.message.match(/(?<=profit":)(-?\d+.\d+)/); - if (profit) { - [ log.profit ] = profit; - } - }); - - const recordsTotal = await this.logsRepository.getTotal(); - return { - draw: request.body.draw, - recordsFiltered: logs.length, - recordsTotal: recordsTotal.total, - data: logs - }; - } - - async getLogsData(request, response){ - datatable(this.logsRepository, reqest.query, {}) - .then((result) => { - // result is response for datatables - return result; - }); - + async getLogsData(request, response) { + return Datatable(this.logsRepository, request.body, undefined, { replaceRegexp: true }); } }; diff --git a/templates/logs.html.twig b/templates/logs.html.twig index 4f4a51da1..0aceb0e84 100644 --- a/templates/logs.html.twig +++ b/templates/logs.html.twig @@ -3,7 +3,7 @@ {% block title %}Logs | Crypto Bot{% endblock %} {% block head %} - + {% endblock %} @@ -35,29 +35,22 @@
- Filters -
+ Filters
-
+
  • Exclude:
    • {% for key,level in levels %}
  • - +
    • {% endfor %}
-
- - -
-
-
@@ -90,6 +83,8 @@ {% block javascript %} {{ parent() }} - + + + {% endblock javascript %} diff --git a/web/static/js/logs.js b/web/static/js/logs.js index e0a895744..8c755c896 100644 --- a/web/static/js/logs.js +++ b/web/static/js/logs.js @@ -1,5 +1,17 @@ $(function() { - const t = $('#logsTable').DataTable({ + function getExcludeFilter() { + // build a regex filter string with an or(|) condition + const positions = $('input:checkbox[name="logExcludeLevels"]:checked') + .map(function() { + return this.value; + }) + .get() + .join('|'); + + return positions ? `^((?!${positions}).*)$` : ''; + } + + const logsTable = $('#logsTable').DataTable({ paging: true, pageLength: 10, processing: true, @@ -8,25 +20,35 @@ $(function() { type: 'POST', url: '/logsTable' }, - // data: [{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.6820223348452448,\"updatedAt\":\"2020-05-06T09:46:55.903Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:46:55.903Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758406\",\"bid\":9078.8,\"ask\":9078.9,\"createdAt\":\"2020-05-06T09:46:46.151Z\"}","created_at":1588758423,"profit":"0.6820223348452448"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.6820223348452448,\"updatedAt\":\"2020-05-06T09:46:25.905Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:46:25.905Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758376\",\"bid\":9078.8,\"ask\":9078.9,\"createdAt\":\"2020-05-06T09:46:16.119Z\"}","created_at":1588758393,"profit":"0.6820223348452448"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.6565158085014344,\"updatedAt\":\"2020-05-06T09:45:55.912Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:45:55.912Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758356\",\"bid\":9076.5,\"ask\":9077,\"createdAt\":\"2020-05-06T09:45:56.173Z\"}","created_at":1588758362,"profit":"0.6565158085014344"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.6609517261264575,\"updatedAt\":\"2020-05-06T09:45:25.900Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:45:25.900Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758326\",\"bid\":9076.9,\"ask\":9077,\"createdAt\":\"2020-05-06T09:45:26.161Z\"}","created_at":1588758331,"profit":"0.6609517261264575"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.6110476528450981,\"updatedAt\":\"2020-05-06T09:44:55.900Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:44:55.900Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758296\",\"bid\":9072.4,\"ask\":9072.5,\"createdAt\":\"2020-05-06T09:44:56.121Z\"}","created_at":1588758300,"profit":"0.6110476528450981"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.5811052088762869,\"updatedAt\":\"2020-05-06T09:44:25.906Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:44:25.906Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758266\",\"bid\":9069.7,\"ask\":9069.8,\"createdAt\":\"2020-05-06T09:44:26.124Z\"}","created_at":1588758269,"profit":"0.5811052088762869"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.5633615383762391,\"updatedAt\":\"2020-05-06T09:43:55.905Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:43:55.905Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758236\",\"bid\":9068.1,\"ask\":9069.7,\"createdAt\":\"2020-05-06T09:43:56.171Z\"}","created_at":1588758239,"profit":"0.5633615383762391"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.5844321470950264,\"updatedAt\":\"2020-05-06T09:43:25.896Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:43:25.896Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758196\",\"bid\":9070,\"ask\":9070.1,\"createdAt\":\"2020-05-06T09:43:16.121Z\"}","created_at":1588758208,"profit":"0.5844321470950264"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.6032847970013355,\"updatedAt\":\"2020-05-06T09:42:55.897Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:42:55.897Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758146\",\"bid\":9071.7,\"ask\":9071.8,\"createdAt\":\"2020-05-06T09:42:26.187Z\"}","created_at":1588758177,"profit":"0.6032847970013355"},{"level":"info","message":"Ticker out of range stop loss (long): {\"symbol\":\"BTCUSD\",\"side\":\"long\",\"amount\":0.15,\"profit\":0.6032847970013355,\"updatedAt\":\"2020-05-06T09:42:25.894Z\",\"entry\":9017.3,\"createdAt\":\"2020-05-06T09:42:25.894Z\"}{\"exchange\":\"bitfinex\",\"symbol\":\"BTCUSD\",\"time\":\"1588758146\",\"bid\":9071.7,\"ask\":9071.8,\"createdAt\":\"2020-05-06T09:42:26.187Z\"}","created_at":1588758146,"profit":"0.6032847970013355"}], columns: [ { data: 'level', name: 'Level' }, - { data: 'message', name: 'Message' }, { - data: 'created_at', - name: 'CreatedAt' - /* , - formatter: function($d, $row) { - return date('jS M y', strtotime($d)); - } */ - } - ], - columnDefs: [ + data: 'message', + name: 'Message', + render: function(data, type, row) { + const profit = data.match(/(?<=profit":)(-?\d+.\d+)/); + if (profit && type === 'display') + return data.replace(profit[0], '' + profit[0]+ '') + else return data; + } + }, { - searchable: true, - orderable: true, - targets: 0 + data: 'createdAt', + name: 'CreatedAt', + render: function(data, type, row) { + return type === 'display' ? moment(data).format('YYYY-MM-DD HH:mm:ss') : data; + } } - ] + ], + order: [[2, 'desc']] + }); + + $('input:checkbox').on('change', function() { + // filter in column 1 by removing checked items, with an regex, no smart filtering, not case sensitive + // Example: ^((?!word1|word2).*)$ + logsTable + .column(0) + .search(getExcludeFilter(), true, false, false) + .draw(false); }); }); From 7069e24da1590c8c05e6444bf513aa27be8da283 Mon Sep 17 00:00:00 2001 From: egisz Date: Thu, 21 May 2020 10:20:34 +0300 Subject: [PATCH 08/25] fix eslint errors --- src/modules/http.js | 4 ++-- src/modules/system/logs_http.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/modules/http.js b/src/modules/http.js index 0cfa1b7eb..5b67857cc 100644 --- a/src/modules/http.js +++ b/src/modules/http.js @@ -160,11 +160,11 @@ module.exports = class Http { }); app.get('/logs', async (req, res) => { - res.render('../templates/logs.html.twig', await this.logsHttp.getLogsPageVariables(req, res)); + res.render('../templates/logs.html.twig', await this.logsHttp.getLogsPageVariables()); }); app.post('/logsTable', async (req, res) => { - res.json(await this.logsHttp.getLogsData(req, res)); + res.json(await this.logsHttp.getLogsData(req)); }); app.get('/desks/:desk', async (req, res) => { diff --git a/src/modules/system/logs_http.js b/src/modules/system/logs_http.js index b5a7d0983..1a967b17c 100644 --- a/src/modules/system/logs_http.js +++ b/src/modules/system/logs_http.js @@ -5,7 +5,7 @@ module.exports = class LogsHttp { this.logsRepository = logsRepository; } - async getLogsPageVariables(request, response) { + async getLogsPageVariables() { return { levels: await this.logsRepository .findAll({ @@ -17,7 +17,7 @@ module.exports = class LogsHttp { }; } - async getLogsData(request, response) { + async getLogsData(request) { return Datatable(this.logsRepository, request.body, undefined, { replaceRegexp: true }); } }; From c75d30ce4721270d7b54728e36f9cebd7fd281ce Mon Sep 17 00:00:00 2001 From: egisz Date: Tue, 26 May 2020 12:19:43 +0300 Subject: [PATCH 09/25] bootstrap as npm packadge --- package-lock.json | 10 ++++++++++ package.json | 4 +++- templates/layout.html.twig | 16 +++++++++++++--- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 33f04e6ae..522a778c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1366,6 +1366,11 @@ } } }, + "bootstrap": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-4.5.0.tgz", + "integrity": "sha512-Z93QoXvodoVslA+PWNdk23Hze4RBYIkpb5h8I2HY2Tu2h7A0LpAgLcyrhrSUyo2/Oxm2l1fRZPs1e5hnxnliXA==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -2010,6 +2015,11 @@ "jquery": ">=1.7" } }, + "datatables.net-plugins": { + "version": "1.10.20", + "resolved": "https://registry.npmjs.org/datatables.net-plugins/-/datatables.net-plugins-1.10.20.tgz", + "integrity": "sha512-rnhNmRHe9UEzvM7gtjBay1QodkWUmxLUhHNbmJMYhhUggjtm+BRSlE0PRilkeUkwckpNWzq+0fPd7/i0fpQgzA==" + }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", diff --git a/package.json b/package.json index 3412a3c93..c83327f1b 100644 --- a/package.json +++ b/package.json @@ -22,13 +22,15 @@ "binance-api-node": "^0.9.19", "bitfinex-api-node": "^3.0.2", "bitmex-realtime-api": "^0.4.0", + "bootstrap": "^4.5.0", "ccxt": "^1.27.89", "coinbase-pro": "^0.9.0", "colors": "^1.3.2", "commander": "^4.1.1", "compression": "^1.7.4", - "datatables.net-bs4": "^1.10.20", "cookie-parser": "^1.4.5", + "datatables.net-bs4": "^1.10.20", + "datatables.net-plugins": "^1.10.20", "express": "^4.16.4", "http": "0.0.0", "lodash": "^4.17.11", diff --git a/templates/layout.html.twig b/templates/layout.html.twig index 6300a6031..f973edb69 100644 --- a/templates/layout.html.twig +++ b/templates/layout.html.twig @@ -17,10 +17,14 @@ scratch. This page gets rid of all links and provides the needed markup only. - - + + + + + + {% block stylesheet %}{% endblock stylesheet %} @@ -106,7 +110,13 @@ scratch. This page gets rid of all links and provides the needed markup only. - + + + + + + + - - {% endblock javascript %} diff --git a/templates/orders/orders.html.twig b/templates/orders/orders.html.twig index 7281e58c6..ff8aca5e6 100644 --- a/templates/orders/orders.html.twig +++ b/templates/orders/orders.html.twig @@ -79,13 +79,11 @@

Orders

- - - +
- +
@@ -93,6 +91,8 @@ + + @@ -109,10 +109,10 @@ {% endif %} + + {% endfor %} diff --git a/templates/pairs.html.twig b/templates/pairs.html.twig index 6408897b1..c50b4b861 100644 --- a/templates/pairs.html.twig +++ b/templates/pairs.html.twig @@ -31,63 +31,8 @@
-
TypeAmount Side CancelExchangeID
- - - + {{ exchange }}{{ order.id }}
- - - - - - - - - - - - - - - {% for pair in pairs %} - - - - - - - - - - - {% endfor %} - -
ExchangeSymbolStateCapitalStrategiesOptionsProcessLimit
{{ pair.exchange }}{{ pair.symbol }}{{ pair.state }}{{ pair.capital }}{{ pair.strategies|json_encode }} - - {% if pair.watchdogs %} - {{ pair.watchdogs|json_encode }} - {% endif %} - - {{ pair.process }} -
-
- {% if pair.has_position|default(false) == false and not pair.process %} - - - - - - - {% endif %} - - {% if pair.has_position|default(false) and not pair.process %} - - - {% endif %} - - {% if pair.process|default %} - - {% endif %} -
-
-
+ +
@@ -101,3 +46,7 @@
{% endblock %} +{% block javascript %} +{{ parent() }} + +{% endblock javascript %} diff --git a/templates/trades.html.twig b/templates/trades.html.twig index b3a260e01..466424cc2 100644 --- a/templates/trades.html.twig +++ b/templates/trades.html.twig @@ -35,71 +35,8 @@
- - - - - - - - - - - - - - - - - - {% for position in positions %} - - - - - - - - - - - - - {% endfor %} - + +
ExchangeSymbolAmountCurrencyProfitEntryUpdatedAtCreatedAtSideAction
{{ position.exchange }}{{ position.position.symbol }} - {{ position.position.amount|price_format }} - - {% if position.currency %}{{ position.currency|price_format }}{% endif %} - - {% if position.position.profit is defined %} - {{ position.position.profit|round(2) }} % - {% endif %} - - {% if position.position.entry %} - {{ position.position.entry|price_format }} - {% endif %} - - {% if position.position.updatedAt %} - {{ position.position.updatedAt|date('d.m.y H:i') }} - {% endif %} - - {% if position.position.createdAt %} - {{ position.position.createdAt|date('d.m.y H:i') }} - {% endif %} - - {% if position.position.side == 'short' %} - - {% elseif position.position.side == 'long' %} - - {% endif %} - -
- -
-
@@ -113,53 +50,8 @@
- - - - - - - - - - - - - - - - - - - - - {% for order in orders %} - - - - - - - - - - - - - - - - {% endfor %} - -
ExchangeSymbolTypeIDPriceAmountRetryOurIdCreatedAtUpdatedAtStatusSideAction
{{ order.exchange }}{{ order.order.symbol }}{{ order.order.type }}{{ order.order.id }}{{ order.order.price }}{{ order.order.amount }}{{ order.order.retry }}{{ order.order.ourId }}{{ order.order.createdAt|date('d.m.y H:i') }}{{ order.order.updateAt|date('d.m.y H:i') }}{{ order.order.status }} - {% if order.order.side == 'buy' %} - - {% elseif order.order.side == 'sell' %} - - {% endif %} - - -
+ +
@@ -173,3 +65,7 @@
{% endblock %} +{% block javascript %} +{{ parent() }} + +{% endblock javascript %} diff --git a/web/static/js/customdatarenderer.js b/web/static/js/customdatarenderer.js new file mode 100644 index 000000000..177a37b5e --- /dev/null +++ b/web/static/js/customdatarenderer.js @@ -0,0 +1,127 @@ +jQuery.fn.dataTable.render.greenRed = function(format = {}) { + return function(d, type, row) { + // Order, search and type get the original data + if (type !== 'display') { + return d; + } + + if (typeof d !== 'number' && typeof d !== 'string') { + return d; + } + const dd = format.style === 'percent' ? parseFloat(d) / 100 : parseFloat(d); + return `${dd.toLocaleString(undefined, format)}`; + }; +}; + +jQuery.fn.dataTable.render.tradingviewLink = function(exchangeColumn) { + return function(d, type, row) { + // Order, search and type get the original data + if (type !== 'display') { + return d; + } + + if (typeof d !== 'string' || !exchangeColumn) { + return d; + } + + return `${d}`; + }; +}; + +jQuery.fn.dataTable.render.highlightProfit = function() { + return function(data, type, row) { + // Order, search and type get the original data + if (type !== 'display') { + return data; + } + + if (typeof data !== 'string') { + return data; + } + + const profit = data.match(/(?<=profit":)(-?\d+.\d+)/); + if (!profit) { + return data; + } + + return data.replace(profit[0], '' + profit[0]+ '') + }; +}; + + +jQuery.fn.dataTable.render.highlightProfit = function() { + return function(data, type, row) { + // Order, search and type get the original data + if (type !== 'display') { + return data; + } + + if (typeof data !== 'string') { + return data; + } + + const profit = data.match(/(?<=profit":)(-?\d+.\d+)/); + if (!profit) { + return data; + } + + return data.replace(profit[0], '' + profit[0]+ '') + }; +}; + +// Renders arrow: green up if data == 'long', else red down. +jQuery.fn.dataTable.render.arrows = function() { + return function(data, type, row) { + // Order, search and type get the original data + if (type !== 'display') { + return data; + } + + if (typeof data !== 'string') { + return data; + } + + return (data === 'short' ? `` : + ``) + }; +}; + +jQuery.fn.dataTable.render.JSON = function() { + return function(data, type, row) { + // Order, search and type get the original data + if (type !== 'display') { + return data; + } + + return JSON.stringify(data); + }; +}; + +// Renders buttons for actions +// Parameters: +// assetType - controller name to handle action +// buttonss - an object containing { action : `ActionTitle`, action2 ....} +// or string value to reference row column containing actions JSON objec. +jQuery.fn.dataTable.render.actionButtons = function(showLable = false) { + return function(data, type, row) { + // Order, search and type get the original data + if (type !== 'display') { + return data; + } + + const btnConfig = { + short: { title: `Limit Short`, btnClass: `btn-danger`, iClass: `fa-shopping-cart` }, + long: { title: `Limit Long`, btnClass: `btn-success`, iClass: `fa-cart-plus` }, + short_market: { title: `Market Short`, btnClass: `btn-danger`, iClass: `fa-shopping-cart` }, + long_market: { title: `Market Long`, btnClass: `btn-success`, iClass: `fa-cart-plus` }, + close: { title: `Limit Close`, btnClass: ``, iClass: `fa-window-close` }, + close_market: { title: `Market Close`, btnClass: ``, iClass: `fa-window-close` }, + cancel: { title: `Close`, btnClass: ``, iClass: `fas fa-window-close text-dark` } + }; + const actions = typeof data == 'string' ? [data] : data; + let results = actions.map(action => + `` + ); + return `
` + results.join('') + `
` + }; +}; diff --git a/web/static/js/logs.js b/web/static/js/logs.js index 8c755c896..33f7ac1aa 100644 --- a/web/static/js/logs.js +++ b/web/static/js/logs.js @@ -21,26 +21,21 @@ $(function() { url: '/logsTable' }, columns: [ - { data: 'level', name: 'Level' }, + { data: 'level', title: 'Level' }, + { data: 'message', title: 'Message' }, + { data: 'createdAt', title: 'CreatedAt' } + ], + order: [[2, 'desc']], + columnDefs: [ { - data: 'message', - name: 'Message', - render: function(data, type, row) { - const profit = data.match(/(?<=profit":)(-?\d+.\d+)/); - if (profit && type === 'display') - return data.replace(profit[0], '' + profit[0]+ '') - else return data; - } + targets: [1], + render: $.fn.dataTable.render.highlightProfit() }, { - data: 'createdAt', - name: 'CreatedAt', - render: function(data, type, row) { - return type === 'display' ? moment(data).format('YYYY-MM-DD HH:mm:ss') : data; - } + targets: [2], + render: $.fn.dataTable.render.moment('YYYY-MM-DDTHH:mm:ss.SSSZ', 'YYYY-MM-DD HH:mm:ss') // 2020-05-21T20:38:11.462Z } - ], - order: [[2, 'desc']] + ] }); $('input:checkbox').on('change', function() { diff --git a/web/static/js/orders.js b/web/static/js/orders.js index c22f50857..20449c558 100644 --- a/web/static/js/orders.js +++ b/web/static/js/orders.js @@ -39,4 +39,43 @@ $(function() { scope.find('#amount').val((value / assetPrice).toFixed(8)); // precision (tick / lot size?) } }); + + const ordersTable = $('#ordersTable').DataTable({ + bFilter: false, + paging: false, + info: false, + columnDefs: [ + { + targets: [5, 6], + visible: false, + searchable: false + } + ] + }); + + // bind button actions + $('#ordersTable tbody').on('click', 'button', function() { + const data = ordersTable.row($(this).parents('tr')).data(); + $.ajax({ + url: `/orders/${data[5]}/${data[6]}`, + type: 'DELETE', + success: function(result) { + // ordersTable.ajax.reload(); + location.reload(); + } + }); + }); + + $('button.cancel-all').on('click', 'button', function() { + const data = ordersTable.rows(0).data(); + $.ajax({ + url: `/orders/${data[5]}`, + type: 'DELETE', + success: function(result) { + // ordersTable.ajax.reload(); + location.reload(); + } + }); + }); + }); diff --git a/web/static/js/pairs.js b/web/static/js/pairs.js new file mode 100644 index 000000000..7e18c5d1c --- /dev/null +++ b/web/static/js/pairs.js @@ -0,0 +1,43 @@ +$(function() { + const pairsTable = $('#pairsTable').DataTable({ + bFilter: false, + paging: false, + info: false, + serverSide: true, + ajax: { + type: 'GET', + url: '/pairs/trade' + }, + columns: [ + { data: 'exchange', title: 'Exchange' }, + { data: 'symbol', title: 'Symbol' }, + { data: 'state', title: 'State' }, + { data: 'capital', title: 'Capital' }, + { data: 'strategies', title: 'Strategies' }, + { data: 'watchdogs', title: 'Options' }, + { data: 'process', title: 'Process', defaultContent: '' }, + { data: 'actions', title: 'Actions' } + // { data: 'actions', visible: false } + ], + order: [[1, 'desc']], + columnDefs: [ + { + targets: 1, + render: $.fn.dataTable.render.tradingviewLink('exchange') + }, + { + targets: [4, 5], + render: $.fn.dataTable.render.JSON() + }, + { + targets: 7, + render: $.fn.dataTable.render.actionButtons() + } + ] + }); + + $('#pairsTable tbody').on('click', 'button', function() { + const data = pairsTable.row($(this).parents('tr')).data(); + $.get(`/pairs/${data.exchange}/${data.symbol}/${this.value}`); + }); +}); diff --git a/web/static/js/trades.js b/web/static/js/trades.js new file mode 100644 index 000000000..b6cdef9d3 --- /dev/null +++ b/web/static/js/trades.js @@ -0,0 +1,126 @@ +$(function() { + const tradesTable = $('#tradesTable').DataTable({ + bFilter: false, + paging: false, + info: false, + serverSide: true, + ajax: { + type: 'GET', + url: '/trades/positions' + }, + columns: [ + { data: 'position.side', title: 'Side' }, + { data: 'exchange', title: 'Exchange' }, + { data: 'position.symbol', title: 'Symbol' }, + { data: 'position.amount', title: 'Amount' }, + { data: 'currency', title: 'Currency' }, + { data: 'position.profit', title: 'Profit', defaultContent: '' }, + { data: 'position.entry', title: 'Entry' }, + { data: 'position.updatedAt', title: 'UpdatedAt' }, + { data: 'position.createdAt', title: 'CreatedAt' }, + { data: 'actions', title: 'Actions' } + ], + order: [[1, 'desc']], + columnDefs: [ + { + targets: 0, + render: $.fn.dataTable.render.arrows() + }, + { + targets: 2, + render: $.fn.dataTable.render.tradingviewLink('exchange') + }, + { + targets: 3, + render: $.fn.dataTable.render.greenRed({ minimumFractionDigits: 2 }) + }, + { + targets: [4, 6], + render: $.fn.dataTable.render.number('', '.', 2) + }, + { + targets: 5, + render: $.fn.dataTable.render.greenRed({ style: 'percent', minimumFractionDigits: 2 }) + }, + { + targets: [7, 8], + render: $.fn.dataTable.render.moment('YYYY-MM-DDTHH:mm:ss.SSSZ', 'YYYY-MM-DD HH:mm:ss') // 2020-05-21T20:38:11.462Z + }, + { + targets: 9, + render: $.fn.dataTable.render.actionButtons() + } + ] + }); + + // bind button actions + $('#tradesTable tbody').on('click', 'button', function() { + const data = tradesTable.row($(this).parents('tr')).data(); + $.get(`/pairs/${data.exchange}/${data.position.symbol}/${this.value}`); + }); + + const ordersTable = $('#ordersTable').DataTable({ + bFilter: false, + paging: false, + info: false, + serverSide: true, + ajax: { + type: 'GET', + url: '/trades/orders' + }, + columns: [ + { data: 'order.side', title: 'Side' }, + { data: 'exchange', title: 'Exchange' }, + { data: 'order.symbol', title: 'Symbol' }, + { data: 'order.amount', title: 'Amount' }, + { data: 'order.price', title: 'Price' }, + { data: 'order.type', title: 'Type' }, + { data: 'order.id', title: 'ID' }, + { data: 'order.retry', title: 'Retry' }, + { data: 'order.ourId', title: 'OurId' }, + { data: 'order.createdAt', title: 'CreatedAt' }, + { data: 'order.updatedAt', title: 'UpdatedAt' }, + { data: 'order.status', title: 'Status' }, + { data: 'actions', title: 'Actions' } + ], + order: [[1, 'desc']], + columnDefs: [ + { + targets: 0, + render: $.fn.dataTable.render.arrows() + }, + { + targets: 2, + render: $.fn.dataTable.render.tradingviewLink('exchange') + }, + { + targets: 3, + render: $.fn.dataTable.render.greenRed({ minimumFractionDigits: 2 }) + }, + { + targets: 4, + render: $.fn.dataTable.render.number('', '.', 2) + }, + { + targets: [9, 10], + render: $.fn.dataTable.render.moment('YYYY-MM-DDTHH:mm:ss.SSSZ', 'YYYY-MM-DD HH:mm:ss') // 2020-05-21T20:38:11.462Z + }, + { + targets: 12, + render: $.fn.dataTable.render.actionButtons() + } + ] + }); + + // bind button actions + $('#ordersTable tbody').on('click', 'button', function() { + const data = ordersTable.row($(this).parents('tr')).data(); + $.ajax({ + url: `/orders/${data.exchange}/${data.order.id}`, + type: 'DELETE', + success: function(result) { + ordersTable.ajax.reload(); + } + }); + }); +}); From 0f20ed1fee7ac23ab1b52c9504b3912af4ab2fc1 Mon Sep 17 00:00:00 2001 From: egisz Date: Tue, 26 May 2020 17:13:53 +0300 Subject: [PATCH 12/25] update dependences, sourch css and js from npm --- package-lock.json | 24 ++++++++++++++++++++++-- package.json | 5 +++-- src/modules/http.js | 13 +++++++++++++ templates/layout.html.twig | 12 +++++++----- 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 522a778c1..388018a79 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2020,6 +2020,25 @@ "resolved": "https://registry.npmjs.org/datatables.net-plugins/-/datatables.net-plugins-1.10.20.tgz", "integrity": "sha512-rnhNmRHe9UEzvM7gtjBay1QodkWUmxLUhHNbmJMYhhUggjtm+BRSlE0PRilkeUkwckpNWzq+0fPd7/i0fpQgzA==" }, + "datatables.net-responsive": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/datatables.net-responsive/-/datatables.net-responsive-2.2.5.tgz", + "integrity": "sha512-AuF28BJRQWfke0cwZwgJB5+WHgoDCDAnW59TJWX4JAXYes3iFrJA6mNHWw46Up3bqUJVI2ZxJoKTGpoEHm5hNA==", + "requires": { + "datatables.net": "^1.10.15", + "jquery": ">=1.7" + } + }, + "datatables.net-responsive-bs4": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/datatables.net-responsive-bs4/-/datatables.net-responsive-bs4-2.2.5.tgz", + "integrity": "sha512-ufUywwBDAyBbUsRJ/QAp6aSSVlZLnNy7BI9cJswqC0y+Si4hwLUTz22kbeoJuCERASPs0aLcx2MaYiKnQi/Euw==", + "requires": { + "datatables.net-bs4": "^1.10.15", + "datatables.net-responsive": "2.2.5", + "jquery": ">=1.7" + } + }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -6515,8 +6534,9 @@ } }, "sequelize-datatables": { - "version": "git+https://git@github.com/egisz/sequelize-datatable-node.git#08124feaf79b7c38b5e39513330d20016c61bb6a", - "from": "git+https://git@github.com/egisz/sequelize-datatable-node.git", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/sequelize-datatables/-/sequelize-datatables-3.0.0.tgz", + "integrity": "sha512-LI27fBBDVxiSZL/LO3eH7d1MeplGR9SiDmT8++Wc1UgQUkggC4xqORlfF1i5okAfOxMhNIXvbLOLimPg+6buHA==", "requires": { "bluebird": "^3.5.1", "lodash": "^4.17.10", diff --git a/package.json b/package.json index c83327f1b..fdbc2e7fe 100644 --- a/package.json +++ b/package.json @@ -29,10 +29,11 @@ "commander": "^4.1.1", "compression": "^1.7.4", "cookie-parser": "^1.4.5", - "datatables.net-bs4": "^1.10.20", "datatables.net-plugins": "^1.10.20", + "datatables.net-responsive-bs4": "^2.2.5", "express": "^4.16.4", "http": "0.0.0", + "jquery": "^3.5.1", "lodash": "^4.17.11", "mocha": "^7.1.2", "moment": "^2.25.3", @@ -46,7 +47,7 @@ "request": "^2.88.2", "sendmail": "^1.4.1", "sequelize": "^5.21.7", - "sequelize-datatables": "git+https://git@github.com/egisz/sequelize-datatable-node.git", + "sequelize-datatables": "^3.0.0", "sqlite3": "^4.2.0", "talib": "^1.0.5", "technicalindicators": "^3.1.0", diff --git a/src/modules/http.js b/src/modules/http.js index f62349857..4a5df948f 100644 --- a/src/modules/http.js +++ b/src/modules/http.js @@ -81,6 +81,7 @@ module.exports = class Http { app.use(cookieParser()); app.use(compression()); app.use(express.static(`${this.projectDir}/web/static`, { maxAge: 3600000 * 24 })); + app.use('/scripts/jquery', express.static(Path.join(this.projectDir, 'node_modules/jquery/dist'))); app.use('/scripts/moment', express.static(Path.join(this.projectDir, '/node_modules/moment/min'))); app.use('/scripts/datatables.net', express.static(Path.join(this.projectDir, '/node_modules/datatables.net/js'))); app.use('/scripts/bootstrap', express.static(Path.join(this.projectDir, '/node_modules/bootstrap/dist/js'))); @@ -97,6 +98,18 @@ module.exports = class Http { '/scripts/datatables.net-plugins/', express.static(Path.join(this.projectDir, '/node_modules/datatables.net-plugins')) ); + app.use( + '/scripts/datatables.net-responsive/', + express.static(Path.join(this.projectDir, '/node_modules/datatables.net-responsive/js')) + ); + app.use( + '/scripts/datatables.net-responsive-bs4/', + express.static(Path.join(this.projectDir, '/node_modules/datatables.net-responsive-bs4/js')) + ); + app.use( + '/css/datatables.net-responsive-bs4/', + express.static(Path.join(this.projectDir, '/node_modules/datatables.net-responsive-bs4/css')) + ); const username = this.systemUtil.getConfig('webserver.username'); const password = this.systemUtil.getConfig('webserver.password'); diff --git a/templates/layout.html.twig b/templates/layout.html.twig index f973edb69..7abaaa018 100644 --- a/templates/layout.html.twig +++ b/templates/layout.html.twig @@ -21,8 +21,9 @@ scratch. This page gets rid of all links and provides the needed markup only. - + + @@ -106,15 +107,16 @@ scratch. This page gets rid of all links and provides the needed markup only. - - + - + + - + + From aaf3333d5972c94c5032f599d4511f9081eeb809 Mon Sep 17 00:00:00 2001 From: egisz Date: Tue, 26 May 2020 17:14:49 +0300 Subject: [PATCH 13/25] fix responsive tables to colapse --- templates/logs.html.twig | 5 +-- templates/pairs.html.twig | 2 +- templates/trades.html.twig | 10 +++--- web/static/css/style.css | 4 +++ web/static/js/orders.js | 1 + web/static/js/pairs.js | 15 ++------ web/static/js/trades.js | 74 ++++++++++---------------------------- 7 files changed, 33 insertions(+), 78 deletions(-) diff --git a/templates/logs.html.twig b/templates/logs.html.twig index f94ca8d92..19f683f7e 100644 --- a/templates/logs.html.twig +++ b/templates/logs.html.twig @@ -50,10 +50,7 @@
-
-
-
- +
diff --git a/templates/pairs.html.twig b/templates/pairs.html.twig index c50b4b861..6143a5fed 100644 --- a/templates/pairs.html.twig +++ b/templates/pairs.html.twig @@ -32,7 +32,7 @@
-
+
diff --git a/templates/trades.html.twig b/templates/trades.html.twig index 466424cc2..5d48dbd85 100644 --- a/templates/trades.html.twig +++ b/templates/trades.html.twig @@ -34,11 +34,9 @@
-
- - -
-
+ + +
@@ -51,7 +49,7 @@
-
+
diff --git a/web/static/css/style.css b/web/static/css/style.css index 7a44a3064..b39e91e70 100644 --- a/web/static/css/style.css +++ b/web/static/css/style.css @@ -53,4 +53,8 @@ nav { -ms-user-select: none; user-select: none; } + +#tradesTable button, #ordersTable button, #pairsTable button { + padding: 0 0 0 3px; +} /*end base*/ diff --git a/web/static/js/orders.js b/web/static/js/orders.js index 20449c558..1ad7acde8 100644 --- a/web/static/js/orders.js +++ b/web/static/js/orders.js @@ -44,6 +44,7 @@ $(function() { bFilter: false, paging: false, info: false, + responsive: true, columnDefs: [ { targets: [5, 6], diff --git a/web/static/js/pairs.js b/web/static/js/pairs.js index 7e18c5d1c..cd73cbe29 100644 --- a/web/static/js/pairs.js +++ b/web/static/js/pairs.js @@ -21,18 +21,9 @@ $(function() { ], order: [[1, 'desc']], columnDefs: [ - { - targets: 1, - render: $.fn.dataTable.render.tradingviewLink('exchange') - }, - { - targets: [4, 5], - render: $.fn.dataTable.render.JSON() - }, - { - targets: 7, - render: $.fn.dataTable.render.actionButtons() - } + { render: $.fn.dataTable.render.tradingviewLink('exchange'), targets: 1 }, + { render: $.fn.dataTable.render.JSON(), targets: [4, 5] }, + { render: $.fn.dataTable.render.actionButtons(), targets: 7 } ] }); diff --git a/web/static/js/trades.js b/web/static/js/trades.js index b6cdef9d3..176f09c2d 100644 --- a/web/static/js/trades.js +++ b/web/static/js/trades.js @@ -3,15 +3,16 @@ $(function() { bFilter: false, paging: false, info: false, + ordering: false, serverSide: true, ajax: { type: 'GET', url: '/trades/positions' }, columns: [ - { data: 'position.side', title: 'Side' }, { data: 'exchange', title: 'Exchange' }, { data: 'position.symbol', title: 'Symbol' }, + { data: 'position.side', title: 'Side' }, { data: 'position.amount', title: 'Amount' }, { data: 'currency', title: 'Currency' }, { data: 'position.profit', title: 'Profit', defaultContent: '' }, @@ -22,34 +23,14 @@ $(function() { ], order: [[1, 'desc']], columnDefs: [ - { - targets: 0, - render: $.fn.dataTable.render.arrows() - }, - { - targets: 2, - render: $.fn.dataTable.render.tradingviewLink('exchange') - }, - { - targets: 3, - render: $.fn.dataTable.render.greenRed({ minimumFractionDigits: 2 }) - }, - { - targets: [4, 6], - render: $.fn.dataTable.render.number('', '.', 2) - }, - { - targets: 5, - render: $.fn.dataTable.render.greenRed({ style: 'percent', minimumFractionDigits: 2 }) - }, - { - targets: [7, 8], - render: $.fn.dataTable.render.moment('YYYY-MM-DDTHH:mm:ss.SSSZ', 'YYYY-MM-DD HH:mm:ss') // 2020-05-21T20:38:11.462Z - }, - { - targets: 9, - render: $.fn.dataTable.render.actionButtons() - } + { render: $.fn.dataTable.render.tradingviewLink('exchange'), targets: 1 }, + { render: $.fn.dataTable.render.arrows(), targets: 2 }, + { render: $.fn.dataTable.render.greenRed({ minimumFractionDigits: 2 }), targets: 3 }, + { render: $.fn.dataTable.render.number('', '.', 2), targets: [4, 6] }, + { render: $.fn.dataTable.render.greenRed({ style: 'percent', minimumFractionDigits: 2 }), targets: 5 }, + { render: $.fn.dataTable.render.moment('YYYY-MM-DDTHH:mm:ss.SSSZ', 'YYYY-MM-DD HH:mm:ss'), targets: [7, 8] }, + { render: $.fn.dataTable.render.actionButtons(), targets: 9 }, + { responsivePriority: 10001, targets: [7, 8] } ] }); @@ -63,15 +44,16 @@ $(function() { bFilter: false, paging: false, info: false, + ordering: false, serverSide: true, ajax: { type: 'GET', url: '/trades/orders' }, columns: [ - { data: 'order.side', title: 'Side' }, { data: 'exchange', title: 'Exchange' }, { data: 'order.symbol', title: 'Symbol' }, + { data: 'order.side', title: 'Side' }, { data: 'order.amount', title: 'Amount' }, { data: 'order.price', title: 'Price' }, { data: 'order.type', title: 'Type' }, @@ -85,30 +67,12 @@ $(function() { ], order: [[1, 'desc']], columnDefs: [ - { - targets: 0, - render: $.fn.dataTable.render.arrows() - }, - { - targets: 2, - render: $.fn.dataTable.render.tradingviewLink('exchange') - }, - { - targets: 3, - render: $.fn.dataTable.render.greenRed({ minimumFractionDigits: 2 }) - }, - { - targets: 4, - render: $.fn.dataTable.render.number('', '.', 2) - }, - { - targets: [9, 10], - render: $.fn.dataTable.render.moment('YYYY-MM-DDTHH:mm:ss.SSSZ', 'YYYY-MM-DD HH:mm:ss') // 2020-05-21T20:38:11.462Z - }, - { - targets: 12, - render: $.fn.dataTable.render.actionButtons() - } + { render: $.fn.dataTable.render.tradingviewLink('exchange'), targets: 1 }, + { render: $.fn.dataTable.render.arrows(), targets: 2 }, + { render: $.fn.dataTable.render.greenRed({ minimumFractionDigits: 2 }), targets: 3 }, + { render: $.fn.dataTable.render.number('', '.', 2), targets: 4 }, + { render: $.fn.dataTable.render.moment('YYYY-MM-DDTHH:mm:ss.SSSZ', 'YYYY-MM-DD HH:mm:ss'), targets: [9, 10] }, + { render: $.fn.dataTable.render.actionButtons(), targets: 12 } ] }); @@ -118,7 +82,7 @@ $(function() { $.ajax({ url: `/orders/${data.exchange}/${data.order.id}`, type: 'DELETE', - success: function(result) { + success: function() { ordersTable.ajax.reload(); } }); From 4b22364548aac75109877e8242f94de675d3d1d7 Mon Sep 17 00:00:00 2001 From: egisz Date: Tue, 26 May 2020 17:50:43 +0300 Subject: [PATCH 14/25] add db migration file and set it run automatically after npm install --- README.md | 5 --- db/migrations/20200526142117-initial.js | 5 +++ package-lock.json | 52 ++++++------------------- package.json | 6 +-- 4 files changed, 19 insertions(+), 49 deletions(-) create mode 100644 db/migrations/20200526142117-initial.js diff --git a/README.md b/README.md index c7d7efeab..8ca61d891 100644 --- a/README.md +++ b/README.md @@ -86,11 +86,6 @@ Provide a configuration with your exchange credentials cp conf.json.dist conf.json ``` -Create a new sqlite database use bot.sql scheme to create the tables -``` -sqlite3 bot.db < bot.sql -``` - Lets start it ``` diff --git a/db/migrations/20200526142117-initial.js b/db/migrations/20200526142117-initial.js new file mode 100644 index 000000000..ec4e578e9 --- /dev/null +++ b/db/migrations/20200526142117-initial.js @@ -0,0 +1,5 @@ +module.exports = { + up: async (queryInterface, Sequelize) => { + await queryInterface.dropAllTables(); + } +}; diff --git a/package-lock.json b/package-lock.json index 388018a79..e57fc18e6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1629,7 +1629,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/cli-color/-/cli-color-1.4.0.tgz", "integrity": "sha512-xu6RvQqqrWEo6MPR1eixqGPywhYBHRs653F9jfXB2Hx4jdM/3WxiNE1vppRmxtMIfl16SFYTpYlrnqH/HsK/2w==", - "dev": true, "requires": { "ansi-regex": "^2.1.1", "d": "1", @@ -1851,7 +1850,6 @@ "version": "1.1.12", "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.12.tgz", "integrity": "sha512-a1eOIcu8+7lUInge4Rpf/n4Krkf3Dd9lqhljRzII1/Zno/kRtUWnznPO3jOKBmTEktkt3fkxisUcivoj0ebzoA==", - "dev": true, "requires": { "ini": "^1.3.4", "proto-list": "~1.2.1" @@ -1978,7 +1976,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", - "dev": true, "requires": { "es5-ext": "^0.10.50", "type": "^1.0.1" @@ -2159,7 +2156,6 @@ "version": "0.15.3", "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz", "integrity": "sha512-M9wIMFx96vq0R4F+gRpY3o2exzb8hEj/n9S8unZtHSvYjibBp/iMufSzvmOcV/laG0ZtuTVGtiJggPOSW2r93g==", - "dev": true, "requires": { "commander": "^2.19.0", "lru-cache": "^4.1.5", @@ -2170,8 +2166,7 @@ "commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", - "dev": true + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" } } }, @@ -2265,7 +2260,6 @@ "version": "0.10.53", "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz", "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==", - "dev": true, "requires": { "es6-iterator": "~2.0.3", "es6-symbol": "~3.1.3", @@ -2276,7 +2270,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", "integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=", - "dev": true, "requires": { "d": "1", "es5-ext": "^0.10.35", @@ -2292,7 +2285,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", - "dev": true, "requires": { "d": "^1.0.1", "ext": "^1.1.2" @@ -2302,7 +2294,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.3.tgz", "integrity": "sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==", - "dev": true, "requires": { "d": "1", "es5-ext": "^0.10.46", @@ -3338,7 +3329,6 @@ "version": "0.3.5", "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", "integrity": "sha1-34xp7vFkeSPHFXuc6DhAYQsCzDk=", - "dev": true, "requires": { "d": "1", "es5-ext": "~0.10.14" @@ -3434,7 +3424,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", "integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==", - "dev": true, "requires": { "type": "^2.0.0" }, @@ -3442,8 +3431,7 @@ "type": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/type/-/type-2.0.0.tgz", - "integrity": "sha512-KBt58xCHry4Cejnc2ISQAF7QY+ORngsWfxezO68+12hKV6lQY8P/psIkcbjeHWn7MqcgciWJyCCevFMJdIXpow==", - "dev": true + "integrity": "sha512-KBt58xCHry4Cejnc2ISQAF7QY+ORngsWfxezO68+12hKV6lQY8P/psIkcbjeHWn7MqcgciWJyCCevFMJdIXpow==" } } }, @@ -4414,8 +4402,7 @@ "is-promise": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", - "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=", - "dev": true + "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=" }, "is-regex": { "version": "1.0.4", @@ -4521,7 +4508,6 @@ "version": "1.11.0", "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.11.0.tgz", "integrity": "sha512-a26B+Cx7USQGSWnz9YxgJNMmML/QG2nqIaL7VVYPCXbqiKz8PN0waSNvroMtvAK6tY7g/wPdNWGEP+JTNIBr6A==", - "dev": true, "requires": { "config-chain": "^1.1.12", "editorconfig": "^0.15.3", @@ -4533,14 +4519,12 @@ "mkdirp": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "dev": true + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" }, "nopt": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", - "dev": true, "requires": { "abbrev": "1", "osenv": "^0.1.4" @@ -4808,7 +4792,6 @@ "version": "4.1.5", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", - "dev": true, "requires": { "pseudomap": "^1.0.2", "yallist": "^2.1.2" @@ -4817,8 +4800,7 @@ "yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", - "dev": true + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" } } }, @@ -4826,7 +4808,6 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/lru-queue/-/lru-queue-0.1.0.tgz", "integrity": "sha1-Jzi9nw089PhEkMVzbEhpmsYyzaM=", - "dev": true, "requires": { "es5-ext": "~0.10.2" } @@ -4900,7 +4881,6 @@ "version": "0.4.14", "resolved": "https://registry.npmjs.org/memoizee/-/memoizee-0.4.14.tgz", "integrity": "sha512-/SWFvWegAIYAO4NQMpcX+gcra0yEZu4OntmUdrBaWrJncxOqAziGFlHxc7yjKVK2uu3lpPW27P27wkR82wA8mg==", - "dev": true, "requires": { "d": "1", "es5-ext": "^0.10.45", @@ -5326,8 +5306,7 @@ "next-tick": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", - "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=", - "dev": true + "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" }, "nice-try": { "version": "1.0.5", @@ -5817,8 +5796,7 @@ "path-parse": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", - "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", - "dev": true + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==" }, "path-to-regexp": { "version": "0.1.7", @@ -5975,8 +5953,7 @@ "proto-list": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", - "integrity": "sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=", - "dev": true + "integrity": "sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=" }, "proxy-addr": { "version": "2.0.5", @@ -5990,8 +5967,7 @@ "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", - "dev": true + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" }, "psl": { "version": "1.6.0", @@ -6327,7 +6303,6 @@ "version": "1.13.1", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.13.1.tgz", "integrity": "sha512-CxqObCX8K8YtAhOBRg+lrcdn+LK+WYOS8tSjqSFbjtrI5PnS63QPhZl4+yKfrU9tdsbMu9Anr/amegT87M9Z6w==", - "dev": true, "requires": { "path-parse": "^1.0.6" } @@ -6521,7 +6496,6 @@ "version": "5.5.1", "resolved": "https://registry.npmjs.org/sequelize-cli/-/sequelize-cli-5.5.1.tgz", "integrity": "sha512-ZM4kUZvY3y14y+Rq3cYxGH7YDJz11jWHcN2p2x7rhAIemouu4CEXr5ebw30lzTBtyXV4j2kTO+nUjZOqzG7k+Q==", - "dev": true, "requires": { "bluebird": "^3.5.3", "cli-color": "^1.4.0", @@ -6619,8 +6593,7 @@ "sigmund": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz", - "integrity": "sha1-P/IfGYytIXX587eBhT/ZTQ0ZtZA=", - "dev": true + "integrity": "sha1-P/IfGYytIXX587eBhT/ZTQ0ZtZA=" }, "signal-exit": { "version": "3.0.2", @@ -7430,7 +7403,6 @@ "version": "0.1.7", "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.7.tgz", "integrity": "sha512-b85NUNzTSdodShTIbky6ZF02e8STtVVfD+fu4aXXShEELpozH+bCpJLYMPZbsABN2wDH7fJpqIoXxJpzbf0NqQ==", - "dev": true, "requires": { "es5-ext": "~0.10.46", "next-tick": "1" @@ -7725,8 +7697,7 @@ "type": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", - "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==", - "dev": true + "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" }, "type-check": { "version": "0.3.2", @@ -7766,7 +7737,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/umzug/-/umzug-2.3.0.tgz", "integrity": "sha512-Z274K+e8goZK8QJxmbRPhl89HPO1K+ORFtm6rySPhFKfKc5GHhqdzD0SGhSWHkzoXasqJuItdhorSvY7/Cgflw==", - "dev": true, "requires": { "bluebird": "^3.7.2" } diff --git a/package.json b/package.json index fdbc2e7fe..c541d8d1f 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "request": "^2.88.2", "sendmail": "^1.4.1", "sequelize": "^5.21.7", + "sequelize-cli": "^5.5.1", "sequelize-datatables": "^3.0.0", "sqlite3": "^4.2.0", "talib": "^1.0.5", @@ -89,7 +90,7 @@ }, "scripts": { "start": "node index.js trade", - "postinstall": "patch-package", + "postinstall": "patch-package && npx sequelize db:migrate", "test": "mocha 'test/**/*.test.js'", "setup": "pm2 deploy ecosystem.config.js production setup", "deploy": "pm2 deploy ecosystem.config.js production" @@ -105,7 +106,6 @@ "eslint-plugin-prettier": "^3.1.3", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-react": "^7.18.0", - "prettier": "^1.19.1", - "sequelize-cli": "^5.5.1" + "prettier": "^1.19.1" } } From f1e5d8a40b6d6d5a80b69873a23080b954745962 Mon Sep 17 00:00:00 2001 From: egisz Date: Wed, 27 May 2020 15:10:02 +0300 Subject: [PATCH 15/25] remove sql script as it is generated by sequelize --- bot.sql | 84 --------------------------------------------------------- 1 file changed, 84 deletions(-) delete mode 100644 bot.sql diff --git a/bot.sql b/bot.sql deleted file mode 100644 index 7b4bd9bba..000000000 --- a/bot.sql +++ /dev/null @@ -1,84 +0,0 @@ -PRAGMA auto_vacuum = INCREMENTAL; - -CREATE TABLE IF NOT EXISTS candlesticks ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - exchange VARCHAR(255) NULL, - symbol VARCHAR(255) NULL, - period VARCHAR(255) NULL, - time INTEGER NULL, - open REAL NULL, - high REAL NULL, - low REAL NULL, - close REAL NULL, - volume REAL NULL -); - -CREATE UNIQUE INDEX unique_candle - ON candlesticks (exchange, symbol, period, time); - -CREATE INDEX time_idx ON candlesticks (time); -CREATE INDEX exchange_symbol_idx ON candlesticks (exchange, symbol); - -CREATE TABLE IF NOT EXISTS candlesticks_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - income_at BIGINT NULL, - exchange VARCHAR(255) NULL, - symbol VARCHAR(255) NULL, - period VARCHAR(255) NULL, - time INTEGER NULL, - open REAL NULL, - high REAL NULL, - low REAL NULL, - close REAL NULL, - volume REAL NULL -); - -CREATE INDEX candle_idx ON candlesticks_log (exchange, symbol, period, time); - -CREATE TABLE IF NOT EXISTS ticker ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - exchange VARCHAR(255) NULL, - symbol VARCHAR(255) NULL, - ask REAL NULL, - bid REAL NULL, - updated_at INT NULL -); - -CREATE UNIQUE INDEX ticker_unique - ON ticker (exchange, symbol); - -CREATE TABLE IF NOT EXISTS ticker_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - exchange VARCHAR(255) NULL, - symbol VARCHAR(255) NULL, - ask REAL NULL, - bid REAL NULL, - income_at BIGINT NULL -); -CREATE INDEX ticker_log_idx ON ticker_log (exchange, symbol); -CREATE INDEX ticker_log_time_idx ON ticker_log (exchange, symbol, income_at); - -CREATE TABLE IF NOT EXISTS signals ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - exchange VARCHAR(255) NULL, - symbol VARCHAR(255) NULL, - ask REAL NULL, - bid REAL NULL, - options TEXT NULL, - side VARCHAR(50) NULL, - strategy VARCHAR(50) NULL, - income_at BIGINT NULL, - state VARCHAR(50) NULL -); -CREATE INDEX symbol_idx ON signals (exchange, symbol); - -CREATE TABLE IF NOT EXISTS logs ( - uuid VARCHAR(64) PRIMARY KEY, - level VARCHAR(32) NOT NULL, - message TEXT NULL, - created_at INT NOT NULL -); - -CREATE INDEX created_at_idx ON logs (created_at); -CREATE INDEX level_created_at_idx ON logs (level, created_at); -CREATE INDEX level_idx ON logs (level); \ No newline at end of file From 8e6673e1252af949c13eabe471ff8c5b44f047ec Mon Sep 17 00:00:00 2001 From: egisz Date: Thu, 28 May 2020 09:47:28 +0300 Subject: [PATCH 16/25] detection of environment --- conf.json.dist | 2 +- src/modules/services.js | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/conf.json.dist b/conf.json.dist index fc7e4f2a0..04767bca8 100644 --- a/conf.json.dist +++ b/conf.json.dist @@ -155,7 +155,7 @@ "operatorsAliases": false }, "test": { - "storage": "bot.db", + "storage": ":memory:", "dialect": "sqlite", "operatorsAliases": false }, diff --git a/src/modules/services.js b/src/modules/services.js index 40103a57c..16a3e6657 100644 --- a/src/modules/services.js +++ b/src/modules/services.js @@ -145,9 +145,10 @@ module.exports = { return db; } + const env = process.env.NODE_ENV || 'development'; const dbSettings = { - dialect: _.get(config, 'production.dialect'), - storage: _.get(config, 'production.storage') + dialect: _.get(config, `${env}.dialect`), + storage: _.get(config, `${env}.storage`) }; const sequelize = new Sequelize(dbSettings.database, dbSettings.user, dbSettings.password, dbSettings); From 0362e3985f5b3e037552d2698aabe05471e9a097 Mon Sep 17 00:00:00 2001 From: egisz Date: Thu, 28 May 2020 09:47:57 +0300 Subject: [PATCH 17/25] fix logs page formatting --- templates/logs.html.twig | 2 +- web/static/js/logs.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/logs.html.twig b/templates/logs.html.twig index 19f683f7e..04c6b240d 100644 --- a/templates/logs.html.twig +++ b/templates/logs.html.twig @@ -50,7 +50,7 @@
-
+
diff --git a/web/static/js/logs.js b/web/static/js/logs.js index 33f7ac1aa..3bd6820b9 100644 --- a/web/static/js/logs.js +++ b/web/static/js/logs.js @@ -14,7 +14,7 @@ $(function() { const logsTable = $('#logsTable').DataTable({ paging: true, pageLength: 10, - processing: true, + responsive: true, serverSide: true, ajax: { type: 'POST', From 077475a960363d324e1abb7c526b6f7d4da10b68 Mon Sep 17 00:00:00 2001 From: egisz Date: Fri, 29 May 2020 11:53:10 +0300 Subject: [PATCH 18/25] fix signal logging - missed out await --- .vscode/launch.json | 2 +- src/modules/listener/tick_listener.js | 8 ++++---- src/modules/repository/signal_repository.js | 20 +++++++------------- src/modules/services.js | 13 +------------ src/modules/signal/signal_http.js | 2 +- src/modules/signal/signal_logger.js | 9 --------- 6 files changed, 14 insertions(+), 40 deletions(-) delete mode 100644 src/modules/signal/signal_logger.js diff --git a/.vscode/launch.json b/.vscode/launch.json index 5fbb5bb1d..471c8c542 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,7 +11,7 @@ "skipFiles": [ "/**" ], - "program": "${workspaceFolder}\\index.js", + "program": "${workspaceFolder}/index.js", "args": ["trade"] }, { diff --git a/src/modules/listener/tick_listener.js b/src/modules/listener/tick_listener.js index f67e4167e..db60f3463 100644 --- a/src/modules/listener/tick_listener.js +++ b/src/modules/listener/tick_listener.js @@ -8,7 +8,7 @@ module.exports = class TickListener { tickers, instances, notifier, - signalLogger, + signalRepository, strategyManager, exchangeManager, pairStateManager, @@ -18,7 +18,7 @@ module.exports = class TickListener { this.tickers = tickers; this.instances = instances; this.notifier = notifier; - this.signalLogger = signalLogger; + this.signalRepository = signalRepository; this.strategyManager = strategyManager; this.exchangeManager = exchangeManager; this.pairStateManager = pairStateManager; @@ -78,7 +78,7 @@ module.exports = class TickListener { this.notifier.send(`[${signal} (${strategyKey})` + `] ${symbol.exchange}:${symbol.symbol} - ${ticker.ask}`); // log signal - this.signalLogger.signal( + await this.signalRepository.insertSignal( symbol.exchange, symbol.symbol, { @@ -142,7 +142,7 @@ module.exports = class TickListener { [new Date().toISOString(), signal, strategyKey, symbol.exchange, symbol.symbol, ticker.ask].join(' ') ); this.notifier.send(`[${signal} (${strategyKey})` + `] ${symbol.exchange}:${symbol.symbol} - ${ticker.ask}`); - this.signalLogger.signal( + await this.signalRepository.insertSignal( symbol.exchange, symbol.symbol, { diff --git a/src/modules/repository/signal_repository.js b/src/modules/repository/signal_repository.js index 1f6995ca0..b603f3d9f 100644 --- a/src/modules/repository/signal_repository.js +++ b/src/modules/repository/signal_repository.js @@ -12,14 +12,6 @@ module.exports = function(sequelize, DataTypes) { type: DataTypes.STRING(255), allowNull: true }, - ask: { - type: DataTypes.REAL, - allowNull: true - }, - bid: { - type: DataTypes.REAL, - allowNull: true - }, options: { type: DataTypes.TEXT, allowNull: true @@ -31,10 +23,6 @@ module.exports = function(sequelize, DataTypes) { strategy: { type: DataTypes.STRING(50), allowNull: true - }, - state: { - type: DataTypes.STRING(50), - allowNull: true } }, { @@ -52,7 +40,13 @@ module.exports = function(sequelize, DataTypes) { }; SignalRepository.insertSignal = async (exchange, symbol, options, side, strategy) => - SignalRepository.create(exchange, symbol, options, side, strategy); + SignalRepository.create({ + exchange: exchange, + symbol: symbol, + options: JSON.stringify(options || {}), + side: side, + strategy: strategy + }); return SignalRepository; }; diff --git a/src/modules/services.js b/src/modules/services.js index 16a3e6657..a417f749a 100644 --- a/src/modules/services.js +++ b/src/modules/services.js @@ -21,7 +21,6 @@ const TickerDatabaseListener = require('../modules/listener/ticker_database_list const ExchangeOrderWatchdogListener = require('../modules/listener/exchange_order_watchdog_listener'); const ExchangePositionWatcher = require('../modules/exchange/exchange_position_watcher'); -const SignalLogger = require('../modules/signal/signal_logger'); const SignalHttp = require('../modules/signal/signal_http'); const StrategyManager = require('./strategy/strategy_manager'); @@ -80,7 +79,6 @@ let tickListener; let createOrderListener; let exchangeOrderWatchdogListener; -let signalLogger; let signalHttp; let signalRepository; @@ -225,7 +223,7 @@ module.exports = { this.getTickers(), this.getInstances(), this.getNotifier(), - this.getSignalLogger(), + this.getSignalRepository(), this.getStrategyManager(), this.getExchangeManager(), this.getPairStateManager(), @@ -262,15 +260,6 @@ module.exports = { return tickerDatabaseListener; }, - getSignalLogger: function() { - if (signalLogger) { - return signalLogger; - } - - signalLogger = new SignalLogger(this.getSignalRepository()); - return signalLogger; - }, - getSignalHttp: function() { if (signalHttp) { return signalHttp; diff --git a/src/modules/signal/signal_http.js b/src/modules/signal/signal_http.js index 4010ef044..2fbb084c6 100644 --- a/src/modules/signal/signal_http.js +++ b/src/modules/signal/signal_http.js @@ -4,6 +4,6 @@ module.exports = class SignalHttp { } async getSignals(since) { - return await this.signalRepository.getSignals(since); + return this.signalRepository.getSignals(since); } }; diff --git a/src/modules/signal/signal_logger.js b/src/modules/signal/signal_logger.js deleted file mode 100644 index 277713369..000000000 --- a/src/modules/signal/signal_logger.js +++ /dev/null @@ -1,9 +0,0 @@ -module.exports = class SignalLogger { - constructor(signalRepository) { - this.signalRepository = signalRepository; - } - - signal(exchange, symbol, options, side, strategy) { - this.signalRepository.insertSignal(exchange, symbol, options, side, strategy); - } -}; From 330c5eecef1584246d82a0f6a663524d0aa703d7 Mon Sep 17 00:00:00 2001 From: egisz Date: Fri, 29 May 2020 16:11:03 +0300 Subject: [PATCH 19/25] fix logs table wrap --- package.json | 2 +- .../repository/candlestick_repository.js | 3 ++- src/modules/services.js | 6 +----- web/static/css/style.css | 6 +++++- web/static/js/logs.js | 20 +++++++++++-------- 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index c541d8d1f..cfff66e41 100644 --- a/package.json +++ b/package.json @@ -105,7 +105,7 @@ "eslint-plugin-node": "^11.1.0", "eslint-plugin-prettier": "^3.1.3", "eslint-plugin-promise": "^4.2.1", - "eslint-plugin-react": "^7.18.0", + "eslint-plugin-react": "^7.20.0", "prettier": "^1.19.1" } } diff --git a/src/modules/repository/candlestick_repository.js b/src/modules/repository/candlestick_repository.js index 86699da09..930f1c7d5 100644 --- a/src/modules/repository/candlestick_repository.js +++ b/src/modules/repository/candlestick_repository.js @@ -48,7 +48,8 @@ module.exports = function(sequelize, DataTypes) { } }, { - tableName: 'candlesticks' + tableName: 'candlesticks', + timestamps: false } ); diff --git a/src/modules/services.js b/src/modules/services.js index a417f749a..4a472dd82 100644 --- a/src/modules/services.js +++ b/src/modules/services.js @@ -144,11 +144,7 @@ module.exports = { } const env = process.env.NODE_ENV || 'development'; - const dbSettings = { - dialect: _.get(config, `${env}.dialect`), - storage: _.get(config, `${env}.storage`) - }; - + const dbSettings = _.get(config, `${env}`); const sequelize = new Sequelize(dbSettings.database, dbSettings.user, dbSettings.password, dbSettings); db = {}; diff --git a/web/static/css/style.css b/web/static/css/style.css index b39e91e70..9388c6884 100644 --- a/web/static/css/style.css +++ b/web/static/css/style.css @@ -56,5 +56,9 @@ nav { #tradesTable button, #ordersTable button, #pairsTable button { padding: 0 0 0 3px; -} +} + +.breakAll { + overflow-wrap: anywhere; +} /*end base*/ diff --git a/web/static/js/logs.js b/web/static/js/logs.js index 3bd6820b9..d1873c7cd 100644 --- a/web/static/js/logs.js +++ b/web/static/js/logs.js @@ -27,17 +27,21 @@ $(function() { ], order: [[2, 'desc']], columnDefs: [ - { - targets: [1], - render: $.fn.dataTable.render.highlightProfit() - }, - { - targets: [2], - render: $.fn.dataTable.render.moment('YYYY-MM-DDTHH:mm:ss.SSSZ', 'YYYY-MM-DD HH:mm:ss') // 2020-05-21T20:38:11.462Z - } + { render: $.fn.dataTable.render.highlightProfit(), targets: 1 }, + { render: $.fn.dataTable.render.moment('YYYY-MM-DDTHH:mm:ss.SSSZ', 'YYYY-MM-DD HH:mm:ss'), targets: 2 }, + { className: 'breakAll', targets: 1 }, + { responsivePriority: 10001, targets: [0, 2] } ] }); + // do not break words on header + logsTable.on('draw', function() { + logsTable + .column(1) + .header() + .classList.remove('breakAll'); + }); + $('input:checkbox').on('change', function() { // filter in column 1 by removing checked items, with an regex, no smart filtering, not case sensitive // Example: ^((?!word1|word2).*)$ From 42d6eaf2e1618ca70df2dbfea8846803bc65820a Mon Sep 17 00:00:00 2001 From: egisz Date: Wed, 3 Jun 2020 11:06:34 +0300 Subject: [PATCH 20/25] fix Candlestick model condition --- src/modules/repository/candlestick_repository.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/repository/candlestick_repository.js b/src/modules/repository/candlestick_repository.js index 930f1c7d5..f472181a2 100644 --- a/src/modules/repository/candlestick_repository.js +++ b/src/modules/repository/candlestick_repository.js @@ -71,7 +71,7 @@ module.exports = function(sequelize, DataTypes) { ...(typeof(start) !== 'undefined') && {[Op.gt]: start}, ...(typeof(end) !== 'undefined') && {[Op.lt]: end} }; - if (Object.keys(timeConditions).length !== 0) { + if (Reflect.ownKeys(timeConditions).length > 0) { whereCondition.time = timeConditions; }; return CandlestickRepository.findAll({ From aeaa943e0e21e1e55ec8c7525980a8062d705300 Mon Sep 17 00:00:00 2001 From: egisz Date: Wed, 3 Jun 2020 11:43:58 +0300 Subject: [PATCH 21/25] fix pairs page error when no watchdogs present --- web/static/js/pairs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/static/js/pairs.js b/web/static/js/pairs.js index cd73cbe29..e2fb068f5 100644 --- a/web/static/js/pairs.js +++ b/web/static/js/pairs.js @@ -14,7 +14,7 @@ $(function() { { data: 'state', title: 'State' }, { data: 'capital', title: 'Capital' }, { data: 'strategies', title: 'Strategies' }, - { data: 'watchdogs', title: 'Options' }, + { data: 'watchdogs', title: 'Options', defaultContent: '' }, { data: 'process', title: 'Process', defaultContent: '' }, { data: 'actions', title: 'Actions' } // { data: 'actions', visible: false } From b9b8e02d6bfbcfeb6211dc1f1d319570c54e7893 Mon Sep 17 00:00:00 2001 From: egisz Date: Mon, 15 Jun 2020 20:20:39 +0300 Subject: [PATCH 22/25] update travis config --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6d9ef085d..351dbfffd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,5 +6,8 @@ node_js: install: - npm install +before_script: +- cp conf.json.dist conf.json + script: - npm test From afda66a7dd1c21452a1fc3ad73998f3635481467 Mon Sep 17 00:00:00 2001 From: egisz Date: Tue, 16 Jun 2020 11:34:51 +0300 Subject: [PATCH 23/25] another fix to travis config --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 351dbfffd..0ccbc640c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,11 +3,11 @@ language: node_js node_js: - '10' +before_install: +- cp conf.json.dist conf.json + install: - npm install -before_script: -- cp conf.json.dist conf.json - script: - npm test From 85e1087b9336506cab4650a7c187fa4cd398af99 Mon Sep 17 00:00:00 2001 From: egisz Date: Tue, 16 Jun 2020 11:50:09 +0300 Subject: [PATCH 24/25] fix failing tests --- test/modules/listener/tick_listener.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/modules/listener/tick_listener.test.js b/test/modules/listener/tick_listener.test.js index 0702abf1b..e6ff6358d 100644 --- a/test/modules/listener/tick_listener.test.js +++ b/test/modules/listener/tick_listener.test.js @@ -11,7 +11,7 @@ describe('#tick listener for order', function() { { get: () => new Ticker() }, {}, { send: () => {} }, - { signal: () => {} }, + { insertSignal: () => {} }, { executeStrategy: async () => { return SignalResult.createSignal('short', {}); @@ -56,7 +56,7 @@ describe('#tick listener for order', function() { {}, { send: () => {} }, { - signal: (exchange, symbol, opts, signal, strategyKey) => { + insertSignal: (exchange, symbol, opts, signal, strategyKey) => { calls.push(exchange, symbol, opts, signal, strategyKey); return []; } From 6643851aab9fa3ef65e63fc57ff4e3728f870306 Mon Sep 17 00:00:00 2001 From: egisz Date: Fri, 3 Jul 2020 14:36:20 +0300 Subject: [PATCH 25/25] fix signals date column --- templates/signals.html.twig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/signals.html.twig b/templates/signals.html.twig index 9f5b4cf1f..159c5663d 100644 --- a/templates/signals.html.twig +++ b/templates/signals.html.twig @@ -59,7 +59,7 @@ {% endif %} {{ signal.strategy }} - {{ signal.income_at|date('d.m.y H:i') }} + {{ signal.updatedAt|date('d.m.y H:i') }} {{ signal.state }} {{ signal.options }}