perf: improve performance mongodb and redis (chrisleekr#288)

aileen5150 · Aug 31, 2021 · b1c4edc · b1c4edc
1 parent 1b072d8
commit b1c4edc
Show file tree

Hide file tree

Showing 54 changed files with 5,958 additions and 5,463 deletions.
diff --git a/.github/workflows/release-and-docker.yml b/.github/workflows/release-and-docker.yml
@@ -56,7 +56,7 @@ jobs:
         with:
           context: .
           file: Dockerfile
-          platforms: linux/amd64,linux/arm/v7
+          platforms: linux/amd64,linux/arm/v7,linux/arm64
           push: true
           target: production-stage
           build-args: |

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,14 @@
 
 All notable changes to this project will be documented in this file.
 
+## [Unreleased]
+
+- Added indexes for MongoDB
+- Updated configuration to use cache as well to improve performance
+- Updated redis.conf to disable AOF for improving performance
+- Moved grid/manual orders to MongoDB from Redis since Redis is not persistent anymore
+- Updated MongoDB to not acknowledge insert/update/delete
+
 ## [0.0.77] - 2021-08-27
 
 - Support setting minimum logging level. Thanks [ruslan-khalitov](https://github.com/ruslan-khalitov) - [#280](https://github.com/chrisleekr/binance-trading-bot/pull/280)

diff --git a/README.ko.md b/README.ko.md
@@ -324,15 +324,15 @@
 
 | Password Protected | Frontend Mobile |
 | ------------------ | --------------- |
-| ![Password Protected](https://user-images.githubusercontent.com/5715919/127773484-51d01881-4933-454e-9052-9965b222e716.png) | ![Frontend Mobile](https://user-images.githubusercontent.com/5715919/129367660-743f89fd-2eae-4113-8ce7-6249ba47926a.png) |
+| ![Password Protected](https://user-images.githubusercontent.com/5715919/127773484-51d01881-4933-454e-9052-9965b222e716.png) | ![Frontend Mobile](https://user-images.githubusercontent.com/5715919/131497627-3c637c9a-f0b8-4e8f-a8d7-6002f1602944.png) |
 
 | Setting | Manual Trade |
 | ------- | ------------ |
 | ![Setting](https://user-images.githubusercontent.com/5715919/127318581-4e422ac9-b145-4e83-a90d-5c05c61d6e2f.png) | ![Manual Trade](https://user-images.githubusercontent.com/5715919/127318630-f2180e1b-3feb-48fa-a083-4cb7f90f743f.png) |
 
 | Frontend Desktop  | Closed Trades |
 | ----------------- | ------------- |
-| ![Frontend Desktop](https://user-images.githubusercontent.com/5715919/129367482-2d88144a-b5e3-49b1-ae11-f21e7251dab0.png) | ![Closed Trades](https://user-images.githubusercontent.com/5715919/129367521-019cf591-26f2-4d69-ac9e-9f631605aab2.png) |
+| ![Frontend Desktop](https://user-images.githubusercontent.com/5715919/131497813-03cef79b-7b43-4e61-90d9-a4b6d7f1a91b.png) | ![Closed Trades](https://user-images.githubusercontent.com/5715919/131497856-28ec42b7-1290-4b74-8644-d74516758a5b.png) |
 
 ### 샘플 거래
 

diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
 # Binance Trading Bot
 
 [![GitHub version](https://img.shields.io/github/package-json/v/chrisleekr/binance-trading-bot)](https://github.com/chrisleekr/binance-trading-bot/releases)
-[![Build](https://github.com/chrisleekr/binance-trading-bot/workflows/main/badge.svg)](https://github.com/chrisleekr/binance-trading-bot/actions?query=workflow%3Amain)
+[![Build](https://github.com/chrisleekr/binance-trading-bot/workflows/Push/badge.svg)](https://github.com/chrisleekr/binance-trading-bot/actions?query=workflow%3APush)
 [![CodeCov](https://codecov.io/gh/chrisleekr/binance-trading-bot/branch/master/graph/badge.svg)](https://codecov.io/gh/chrisleekr/binance-trading-bot)
 [![Docker pull](https://img.shields.io/docker/pulls/chrisleekr/binance-trading-bot)](https://hub.docker.com/r/chrisleekr/binance-trading-bot)
 [![GitHub contributors](https://img.shields.io/github/contributors/chrisleekr/binance-trading-bot)](https://github.com/chrisleekr/binance-trading-bot/graphs/contributors)
@@ -348,15 +348,15 @@ Or use the frontend to adjust configurations after launching the application.
 
 | Password Protected | Frontend Mobile |
 | ------------------ | --------------- |
-| ![Password Protected](https://user-images.githubusercontent.com/5715919/127773484-51d01881-4933-454e-9052-9965b222e716.png) | ![Frontend Mobile](https://user-images.githubusercontent.com/5715919/129367660-743f89fd-2eae-4113-8ce7-6249ba47926a.png) |
+| ![Password Protected](https://user-images.githubusercontent.com/5715919/127773484-51d01881-4933-454e-9052-9965b222e716.png) | ![Frontend Mobile](https://user-images.githubusercontent.com/5715919/131497627-3c637c9a-f0b8-4e8f-a8d7-6002f1602944.png) |
 
 | Setting | Manual Trade |
 | ------- | ------------ |
 | ![Setting](https://user-images.githubusercontent.com/5715919/127318581-4e422ac9-b145-4e83-a90d-5c05c61d6e2f.png) | ![Manual Trade](https://user-images.githubusercontent.com/5715919/127318630-f2180e1b-3feb-48fa-a083-4cb7f90f743f.png) |
 
 | Frontend Desktop  | Closed Trades |
 | ----------------- | ------------- |
-| ![Frontend Desktop](https://user-images.githubusercontent.com/5715919/129367482-2d88144a-b5e3-49b1-ae11-f21e7251dab0.png) | ![Closed Trades](https://user-images.githubusercontent.com/5715919/129367521-019cf591-26f2-4d69-ac9e-9f631605aab2.png) |
+| ![Frontend Desktop](https://user-images.githubusercontent.com/5715919/131497813-03cef79b-7b43-4e61-90d9-a4b6d7f1a91b.png) | ![Closed Trades](https://user-images.githubusercontent.com/5715919/131497856-28ec42b7-1290-4b74-8644-d74516758a5b.png) |
 
 ### Sample Trade
 
@@ -370,6 +370,7 @@ Please refer
 [CHANGELOG.md](https://github.com/chrisleekr/binance-trading-bot/blob/master/CHANGELOG.md)
 to view the past changes.
 
+- [ ] Clear exchange/symbol info cache in the Redis periodically - [#284](https://github.com/chrisleekr/binance-trading-bot/issues/284)
 - [ ] Improve sell strategy with conditional stop price percentage based on the profit percentage - [#94](https://github.com/chrisleekr/binance-trading-bot/issues/94)
 - [ ] Add sudden drop buy strategy - [#67](https://github.com/chrisleekr/binance-trading-bot/issues/67)
 - [ ] Add minimum required order amount - [#84](https://github.com/chrisleekr/binance-trading-bot/issues/84)

diff --git a/README.zh-cn.md b/README.zh-cn.md
@@ -318,15 +318,15 @@ The final profit would be
 
 | Password Protected | Frontend Mobile |
 | ------------------ | --------------- |
-| ![Password Protected](https://user-images.githubusercontent.com/5715919/127773484-51d01881-4933-454e-9052-9965b222e716.png) | ![Frontend Mobile](https://user-images.githubusercontent.com/5715919/129367660-743f89fd-2eae-4113-8ce7-6249ba47926a.png) |
+| ![Password Protected](https://user-images.githubusercontent.com/5715919/127773484-51d01881-4933-454e-9052-9965b222e716.png) | ![Frontend Mobile](https://user-images.githubusercontent.com/5715919/131497627-3c637c9a-f0b8-4e8f-a8d7-6002f1602944.png) |
 
 | Setting | Manual Trade |
 | ------- | ------------ |
 | ![Setting](https://user-images.githubusercontent.com/5715919/127318581-4e422ac9-b145-4e83-a90d-5c05c61d6e2f.png) | ![Manual Trade](https://user-images.githubusercontent.com/5715919/127318630-f2180e1b-3feb-48fa-a083-4cb7f90f743f.png) |
 
 | Frontend Desktop  | Closed Trades |
 | ----------------- | ------------- |
-| ![Frontend Desktop](https://user-images.githubusercontent.com/5715919/129367482-2d88144a-b5e3-49b1-ae11-f21e7251dab0.png) | ![Closed Trades](https://user-images.githubusercontent.com/5715919/129367521-019cf591-26f2-4d69-ac9e-9f631605aab2.png) |
+| ![Frontend Desktop](https://user-images.githubusercontent.com/5715919/131497813-03cef79b-7b43-4e61-90d9-a4b6d7f1a91b.png) | ![Closed Trades](https://user-images.githubusercontent.com/5715919/131497856-28ec42b7-1290-4b74-8644-d74516758a5b.png) |
 
 ### Sample Trade
 

diff --git a/app/cronjob/__tests__/trailingTrade.test.js b/app/cronjob/__tests__/trailingTrade.test.js
@@ -1,3 +1,4 @@
+/* eslint-disable max-classes-per-file */
 /* eslint-disable global-require */
 const { logger } = require('../../helpers');
 
@@ -21,7 +22,6 @@ describe('trailingTrade', () => {
   let mockGetSymbolInfo;
   let mockGetOverrideAction;
   let mockEnsureManualOrder;
-  let mockEnsureOrderPlaced;
   let mockEnsureGridTradeOrderExecuted;
   let mockGetBalances;
   let mockGetOpenOrders;
@@ -130,13 +130,6 @@ describe('trailingTrade', () => {
           }
         }));
 
-      mockEnsureOrderPlaced = jest
-        .fn()
-        .mockImplementation((_logger, rawData) => ({
-          ...rawData,
-          ensure: 'order-placed'
-        }));
-
       mockEnsureGridTradeOrderExecuted = jest
         .fn()
         .mockImplementation((_logger, rawData) => ({
@@ -285,7 +278,6 @@ describe('trailingTrade', () => {
         getSymbolInfo: mockGetSymbolInfo,
         getOverrideAction: mockGetOverrideAction,
         ensureManualOrder: mockEnsureManualOrder,
-        ensureOrderPlaced: mockEnsureOrderPlaced,
         ensureGridTradeOrderExecuted: mockEnsureGridTradeOrderExecuted,
         getBalances: mockGetBalances,
         getOpenOrders: mockGetOpenOrders,
@@ -343,7 +335,6 @@ describe('trailingTrade', () => {
             saveToCache: true,
             overrideAction: { action: 'override-action' },
             ensureManualOrder: { ensured: 'manual-buy-order' },
-            ensure: 'order-placed',
             ensureGridTradeOrder: { ensured: 'grid-trade' },
             handled: 'open-orders',
             placeManualTrade: { placed: 'manual-trade' },
@@ -380,7 +371,6 @@ describe('trailingTrade', () => {
             saveToCache: true,
             overrideAction: { action: 'override-action' },
             ensureManualOrder: { ensured: 'manual-buy-order' },
-            ensure: 'order-placed',
             ensureGridTradeOrder: { ensured: 'grid-trade' },
             handled: 'open-orders',
             placeManualTrade: { placed: 'manual-trade' },
@@ -417,7 +407,6 @@ describe('trailingTrade', () => {
             saveToCache: true,
             overrideAction: { action: 'override-action' },
             ensureManualOrder: { ensured: 'manual-buy-order' },
-            ensure: 'order-placed',
             ensureGridTradeOrder: { ensured: 'grid-trade' },
             handled: 'open-orders',
             placeManualTrade: { placed: 'manual-trade' },
@@ -505,13 +494,6 @@ describe('trailingTrade', () => {
           }
         }));
 
-      mockEnsureOrderPlaced = jest
-        .fn()
-        .mockImplementation((_logger, rawData) => ({
-          ...rawData,
-          ensure: 'order-placed'
-        }));
-
       mockEnsureGridTradeOrderExecuted = jest
         .fn()
         .mockImplementation((_logger, rawData) => ({
@@ -660,7 +642,6 @@ describe('trailingTrade', () => {
         getSymbolInfo: mockGetSymbolInfo,
         getOverrideAction: mockGetOverrideAction,
         ensureManualOrder: mockEnsureManualOrder,
-        ensureOrderPlaced: mockEnsureOrderPlaced,
         ensureGridTradeOrderExecuted: mockEnsureGridTradeOrderExecuted,
         getBalances: mockGetBalances,
         getOpenOrders: mockGetOpenOrders,
@@ -708,7 +689,6 @@ describe('trailingTrade', () => {
             symbolInfo: { symbol: 'info' },
             overrideAction: { action: 'override-action' },
             ensureManualOrder: { ensured: 'manual-buy-order' },
-            ensure: 'order-placed',
             ensureGridTradeOrder: { ensured: 'grid-trade' },
             baseAssetBalance: { baseAsset: 'balance' },
             quoteAssetBalance: { quoteAsset: 'balance' },
@@ -745,7 +725,6 @@ describe('trailingTrade', () => {
             symbolInfo: { symbol: 'info' },
             overrideAction: { action: 'override-action' },
             ensureManualOrder: { ensured: 'manual-buy-order' },
-            ensure: 'order-placed',
             ensureGridTradeOrder: { ensured: 'grid-trade' },
             baseAssetBalance: { baseAsset: 'balance' },
             quoteAssetBalance: { quoteAsset: 'balance' },
@@ -782,7 +761,6 @@ describe('trailingTrade', () => {
             symbolInfo: { symbol: 'info' },
             overrideAction: { action: 'override-action' },
             ensureManualOrder: { ensured: 'manual-buy-order' },
-            ensure: 'order-placed',
             ensureGridTradeOrder: { ensured: 'grid-trade' },
             baseAssetBalance: { baseAsset: 'balance' },
             quoteAssetBalance: { quoteAsset: 'balance' },
@@ -822,7 +800,7 @@ describe('trailingTrade', () => {
       mockGetSymbolInfo = jest.fn().mockResolvedValue(true);
       mockGetOverrideAction = jest.fn().mockResolvedValue(true);
       mockEnsureManualOrder = jest.fn().mockResolvedValue(true);
-      mockEnsureOrderPlaced = jest.fn().mockResolvedValue(true);
+      mockEnsureGridTradeOrderExecuted = jest.fn().mockResolvedValue(true);
       mockGetBalances = jest.fn().mockResolvedValue(true);
       mockGetOpenOrders = jest.fn().mockResolvedValue(true);
       mockGetIndicators = jest.fn().mockResolvedValue(true);
@@ -854,7 +832,7 @@ describe('trailingTrade', () => {
         getSymbolInfo: mockGetSymbolInfo,
         getOverrideAction: mockGetOverrideAction,
         ensureManualOrder: mockEnsureManualOrder,
-        ensureOrderPlaced: mockEnsureOrderPlaced,
+        ensureGridTradeOrderExecuted: mockEnsureGridTradeOrderExecuted,
         getBalances: mockGetBalances,
         getOpenOrders: mockGetOpenOrders,
         getIndicators: mockGetIndicators,
@@ -947,5 +925,33 @@ describe('trailingTrade', () => {
         }
       });
     });
+
+    describe(`redlock error`, () => {
+      beforeEach(async () => {
+        mockConfigGet = jest.fn(_key => null);
+
+        jest.mock('config', () => ({
+          get: mockConfigGet
+        }));
+
+        mockGetGlobalConfiguration = jest.fn().mockRejectedValueOnce(
+          new (class CustomError extends Error {
+            constructor() {
+              super();
+              this.code = 500;
+              this.message = `redlock:lock-XRPBUSD`;
+            }
+          })()
+        );
+
+        const { execute: trailingTradeExecute } = require('../trailingTrade');
+
+        await trailingTradeExecute(logger);
+      });
+
+      it('does not trigger slack.sendMessagage', () => {
+        expect(mockSlackSendMessage).not.toHaveBeenCalled();
+      });
+    });
   });
 });
diff --git a/app/cronjob/__tests__/trailingTradeIndicator.test.js b/app/cronjob/__tests__/trailingTradeIndicator.test.js
@@ -1,3 +1,4 @@
+/* eslint-disable max-classes-per-file */
 /* eslint-disable global-require */
 const { logger } = require('../../helpers');
 
@@ -449,5 +450,35 @@ describe('trailingTradeIndicator', () => {
         }
       });
     });
+
+    describe(`redlock error`, () => {
+      beforeEach(async () => {
+        mockConfigGet = jest.fn(_key => null);
+
+        jest.mock('config', () => ({
+          get: mockConfigGet
+        }));
+
+        mockGetGlobalConfiguration = jest.fn().mockRejectedValueOnce(
+          new (class CustomError extends Error {
+            constructor() {
+              super();
+              this.code = 500;
+              this.message = `redlock:lock-XRPBUSD`;
+            }
+          })()
+        );
+
+        const {
+          execute: trailingTradeIndicatorExecute
+        } = require('../trailingTradeIndicator');
+
+        await trailingTradeIndicatorExecute(logger);
+      });
+
+      it('does not trigger slack.sendMessagage', () => {
+        expect(mockSlackSendMessage).not.toHaveBeenCalled();
+      });
+    });
   });
 });
diff --git a/app/cronjob/trailingTrade.js b/app/cronjob/trailingTrade.js
@@ -18,7 +18,6 @@ const {
   getSymbolInfo,
   getOverrideAction,
   ensureManualOrder,
-  ensureOrderPlaced,
   ensureGridTradeOrderExecuted,
   getBalances,
   getOpenOrders,
@@ -101,10 +100,6 @@ const execute = async logger => {
             stepName: 'ensure-manual-order',
             stepFunc: ensureManualOrder
           },
-          {
-            stepName: 'ensure-open-placed',
-            stepFunc: ensureOrderPlaced
-          },
           {
             stepName: 'ensure-grid-trade-order-executed',
             stepFunc: ensureGridTradeOrderExecuted
@@ -192,6 +187,12 @@ const execute = async logger => {
       })
     );
   } catch (err) {
+    // For the redlock fail
+    if (err.message.includes('redlock')) {
+      // Simply ignore
+      return;
+    }
+
     logger.error(
       { err, errorCode: err.code, debug: true },
       `⚠ Execution failed.`
@@ -200,8 +201,7 @@ const execute = async logger => {
       err.code === -1001 ||
       err.code === -1021 || // Timestamp for this request is outside of the recvWindow
       err.code === 'ECONNRESET' ||
-      err.code === 'ECONNREFUSED' ||
-      err.message.includes('redlock') // For the redlock fail
+      err.code === 'ECONNREFUSED'
     ) {
       // Let's silent for internal server error or assumed temporary errors
     } else {

diff --git a/app/cronjob/trailingTrade/__tests__/steps.test.js b/app/cronjob/trailingTrade/__tests__/steps.test.js
@@ -8,7 +8,6 @@ describe('steps.js', () => {
       getBalances: expect.any(Function),
       getOverrideAction: expect.any(Function),
       ensureManualOrder: expect.any(Function),
-      ensureOrderPlaced: expect.any(Function),
       ensureGridTradeOrderExecuted: expect.any(Function),
       getOpenOrders: expect.any(Function),
       getIndicators: expect.any(Function),