fix: download log file directly (chrisleekr#561)

.gitignore

@@ -23,7 +23,6 @@ yarn-error.log*
 
 coverage
 build
-data
 log
 
 .env
@@ -32,6 +31,8 @@ log
 
 /public/dist/*
 !/public/dist/.gitkeep
+/public/data/*
+!/public/data/.gitkeep
 
 result.csv
 .~lock.result.csv#

CHANGELOG.md

@@ -2,12 +2,17 @@
 
 All notable changes to this project will be documented in this file.
 
+## Unreleased
+
+- Fixed the issue that cannot export huge logs - [#561](https://github.com/chrisleekr/binance-trading-bot/pull/561)
+
 ## [0.0.96] - 2022-12-28
 
 - Enhanced the suggested break-even amount a grid calculator by [@rando128](https://github.com/rando128) - [#555](https://github.com/chrisleekr/binance-trading-bot/pull/555), [#557](https://github.com/chrisleekr/binance-trading-bot/pull/557)
 - Fixed API error page priority by [@habibalkhabbaz](https://github.com/habibalkhabbaz) - [#556](https://github.com/chrisleekr/binance-trading-bot/pull/556)
 
 Thanks[@rando128](https://github.com/rando128) and [@habibalkhabbaz](https://github.com/habibalkhabbaz) for your great contributions. 💯 :heart:
+
 ## [0.0.95] - 2022-12-19
 
 - Added API error message when using the wrong API key/secret - [#544](https://github.com/chrisleekr/binance-trading-bot/pull/544)

app/__tests__/server-frontend.test.js

@@ -25,6 +25,7 @@ describe('server-frontend', () => {
 
   let mockRateLimiterMiddlewareReq;
   let mockRateLimiterMiddlewareResSend;
+  let mockRateLimiterMiddlewareResAttachment;
   let mockRateLimiterMiddlewareResStatus;
   let mockRateLimiterMiddlewareRes;
   let mockRateLimiterMiddlewareNext;
@@ -33,8 +34,11 @@ describe('server-frontend', () => {
   let cacheMock;
   let loggerMock;
 
+  let reqPath;
+
   beforeEach(() => {
     jest.clearAllMocks().resetModules();
+    reqPath = '';
 
     config = require('config');
 
@@ -72,8 +76,11 @@ describe('server-frontend', () => {
     mockRateLimiterMiddlewareResStatus = jest.fn().mockImplementation(() => ({
       send: mockRateLimiterMiddlewareResSend
     }));
+    mockRateLimiterMiddlewareResAttachment = jest.fn().mockReturnValue(true);
+
     mockRateLimiterMiddlewareRes = {
-      status: mockRateLimiterMiddlewareResStatus
+      status: mockRateLimiterMiddlewareResStatus,
+      attachment: mockRateLimiterMiddlewareResAttachment
     };
 
     mockRateLimiterMiddlewareNext = jest.fn().mockReturnValue(true);
@@ -85,6 +92,14 @@ describe('server-frontend', () => {
         mockCors();
       } else if (fn.name === 'fileUpload') {
         mockFileUpload();
+      } else if (fn.name === 'attachmentMiddleware') {
+        await fn(
+          {
+            path: reqPath
+          },
+          mockRateLimiterMiddlewareRes,
+          mockRateLimiterMiddlewareNext
+        );
       } else if (fn.name === 'rateLimiterMiddleware') {
         await fn(
           mockRateLimiterMiddlewareReq,
@@ -314,9 +329,28 @@ describe('server-frontend', () => {
             expect.stringContaining(`You are blocked`)
           );
         });
+      });
+
+      describe('when request is for data log', () => {
+        beforeEach(() => {
+          reqPath = '/data/logs/BTC.json';
+          mockRateLimiterRedisGet = jest
+            .fn()
+            .mockReturnValue({ remainingPoints: 5 });
+          mockRateLimiterRedis = jest.fn().mockImplementation(() => ({
+            get: mockRateLimiterRedisGet
+          }));
+
+          jest.mock('rate-limiter-flexible', () => ({
+            RateLimiterRedis: mockRateLimiterRedis
+          }));
+
+          const { runFrontend } = require('../server-frontend');
+          runFrontend(loggerMock);
+        });
 
-        it('does not trigger next', () => {
-          expect(mockRateLimiterMiddlewareNext).not.toHaveBeenCalled();
+        it('triggers res.attachment', () => {
+          expect(mockRateLimiterMiddlewareResAttachment).toHaveBeenCalled();
         });
       });
     });

app/frontend/webserver/handlers/__tests__/grid-trade-logs-export.test.js

@@ -1,15 +1,16 @@
 /* eslint-disable global-require */
-const { tmpdir: tmpDirectory } = require('os');
+const fs = require('fs');
+const path = require('path');
 const { sep: directorySeparator } = require('path');
-const _ = require('lodash');
 
 describe('webserver/handlers/grid-trade-logs-export', () => {
   let loggerMock;
   let mongoMock;
 
-  let rsDownload;
   let rsSend;
 
+  let fileFolder;
+
   const appMock = {
     route: null
   };
@@ -22,10 +23,10 @@ describe('webserver/handlers/grid-trade-logs-export', () => {
     jest.clearAllMocks().resetModules();
 
     rsSend = jest.fn().mockResolvedValue(true);
-    rsDownload = jest.fn().mockResolvedValue(true);
+
     appMock.route = jest.fn(() => ({
       post: jest.fn().mockImplementation(func => {
-        func(postReq, { send: rsSend, download: rsDownload });
+        func(postReq, { send: rsSend });
       })
     }));
   });
@@ -93,6 +94,10 @@ describe('webserver/handlers/grid-trade-logs-export', () => {
     ].forEach(t => {
       describe(`found rows - ${t.desc}`, () => {
         beforeEach(async () => {
+          // Delete log folder
+          fileFolder = path.join(__dirname, '/../../../../../public/data/logs');
+          fs.rmSync(fileFolder, { recursive: true, force: true });
+
           const { logger, mongo } = require('../../../../helpers');
 
           loggerMock = logger;
@@ -133,19 +138,35 @@ describe('webserver/handlers/grid-trade-logs-export', () => {
         });
 
         it('return data', () => {
-          const tempDirLocation = _.escapeRegExp(
-            `${tmpDirectory()}${directorySeparator}`
-          );
-
-          expect(rsDownload).toHaveBeenCalledWith(
-            expect.stringMatching(`${tempDirLocation}(.+).json`)
-          );
+          expect(rsSend).toHaveBeenCalledWith({
+            success: true,
+            status: 200,
+            message: 'Exported log file',
+            data: {
+              fileName: expect.any(String)
+            }
+          });
         });
       });
     });
 
     describe(`cannot find rows`, () => {
       beforeEach(async () => {
+        // Delete log folder
+        fileFolder = path.join(__dirname, '/../../../../../public/data/logs');
+        fs.rmSync(fileFolder, { recursive: true, force: true });
+
+        if (!fs.existsSync(fileFolder)) {
+          fs.mkdirSync(fileFolder);
+        }
+
+        // Create 10 files to test keeping last 10 logs
+        for (let i = 1; i <= 10; i += 1) {
+          fs.writeFileSync(
+            `${fileFolder}${directorySeparator}file${i}.json`,
+            JSON.stringify([{ log: 'value' }])
+          );
+        }
         const { logger, mongo } = require('../../../../helpers');
 
         loggerMock = logger;
@@ -174,6 +195,11 @@ describe('webserver/handlers/grid-trade-logs-export', () => {
         await handleGridTradeLogsExport(loggerMock, appMock);
       });
 
+      it('keeps 10 logs in the folder', () => {
+        const files = fs.readdirSync(fileFolder);
+        expect(files.length).toBe(10);
+      });
+
       it('triggers mongo.findAll', () => {
         expect(mongoMock.findAll).toHaveBeenCalledWith(
           loggerMock,
@@ -188,13 +214,14 @@ describe('webserver/handlers/grid-trade-logs-export', () => {
       });
 
       it('return data', () => {
-        const tempDirLocation = _.escapeRegExp(
-          `${tmpDirectory()}${directorySeparator}`
-        );
-
-        expect(rsDownload).toHaveBeenCalledWith(
-          expect.stringMatching(`${tempDirLocation}(.+).json`)
-        );
+        expect(rsSend).toHaveBeenCalledWith({
+          success: true,
+          status: 200,
+          message: 'Exported log file',
+          data: {
+            fileName: expect.any(String)
+          }
+        });
       });
     });
   });

app/frontend/webserver/handlers/grid-trade-logs-export.js

@@ -1,14 +1,33 @@
-const { v4: uuidv4 } = require('uuid');
+const moment = require('moment-timezone');
 const fs = require('fs');
-
-const { tmpdir: tmpDirectory } = require('os');
+const path = require('path');
 const { sep: directorySeparator } = require('path');
 const {
   verifyAuthenticated
 } = require('../../../cronjob/trailingTradeHelper/common');
 
 const { mongo } = require('../../../helpers');
 
+const keepLastLogs = (fileFolder, numberOfFilesToKeep) => {
+  const recentFiles = fs
+    .readdirSync(fileFolder)
+    .filter(file => fs.lstatSync(path.join(fileFolder, file)).isFile())
+    .map(file => ({
+      file,
+      mtime: fs.lstatSync(path.join(fileFolder, file)).mtime
+    }))
+    .sort((a, b) => b.mtime.getTime() - a.mtime.getTime());
+
+  if (recentFiles.length < numberOfFilesToKeep) {
+    return;
+  }
+  const deleteFiles = recentFiles.slice(numberOfFilesToKeep - 1);
+
+  deleteFiles.forEach(f =>
+    fs.unlinkSync(`${fileFolder}${directorySeparator}${f.file}`)
+  );
+};
+
 const handleGridTradeLogsExport = async (funcLogger, app) => {
   const logger = funcLogger.child({
     endpoint: '/grid-trade-logs-export'
@@ -44,10 +63,26 @@ const handleGridTradeLogsExport = async (funcLogger, app) => {
     const rows = await mongo.findAll(logger, 'trailing-trade-logs', match, {
       sort: { loggedAt: -1 }
     });
-    const filePath = `${tmpDirectory()}${directorySeparator}${uuidv4()}.json`;
+    const fileName = `${symbol}-${moment().format('YYYY-MM-DD-HH-MM-ss')}.json`;
+    const fileFolder = path.join(__dirname, '/../../../../public/data/logs');
+
+    if (!fs.existsSync(fileFolder)) {
+      fs.mkdirSync(fileFolder);
+    }
+
+    keepLastLogs(fileFolder, 10);
+
+    const filePath = `${fileFolder}${directorySeparator}${fileName}`;
     fs.writeFileSync(filePath, JSON.stringify(rows));
 
-    return res.download(filePath);
+    return res.send({
+      success: true,
+      status: 200,
+      message: 'Exported log file',
+      data: {
+        fileName
+      }
+    });
   });
 };
 

app/server-frontend.js

@@ -46,6 +46,12 @@ const runFrontend = async serverLogger => {
       tempFileDir: '/tmp/'
     })
   );
+  // Make data folder to be downloadable
+  const attachmentMiddleware = async (req, res, next) => {
+    if (req.path.split('/')[1] === 'data') res.attachment(); // short for res.set('Content-Disposition', 'attachment')
+    next();
+  };
+  app.use(attachmentMiddleware);
   app.use(express.static(path.join(__dirname, '/../public')));
 
   // Must configure bull board before listen.

nodemon.json

@@ -0,0 +1,3 @@
+{
+  "ignore": ["public/data/*"]
+}

public/js/SymbolLogsIcon.js

@@ -103,37 +103,17 @@ class SymbolLogsIcon extends React.Component {
     return axios
       .post('/grid-trade-logs-export', {
         authToken,
-        symbol,
-        responseType: 'blob'
+        symbol
       })
       .then(response => {
-        const filename = `${symbol}-${moment().format(
-          'YYYY-MM-DD-HH-MM-SS'
-        )}.json`;
-        const contentType = 'application/json;charset=utf-8;';
-        if (window.navigator && window.navigator.msSaveOrOpenBlob) {
-          const blob = new Blob(
-            [
-              decodeURIComponent(
-                encodeURI(JSON.stringify(response.data, null, 2))
-              )
-            ],
-            { type: contentType }
-          );
-          navigator.msSaveOrOpenBlob(blob, filename);
-        } else {
-          const a = document.createElement('a');
-          a.download = filename;
-          a.href =
-            'data:' +
-            contentType +
-            ',' +
-            encodeURIComponent(JSON.stringify(response.data, null, 2));
-          a.target = '_blank';
-          document.body.appendChild(a);
-          a.click();
-          document.body.removeChild(a);
-        }
+        const {
+          data: {
+            data: { fileName }
+          }
+        } = response;
+
+        const filePath = `data/logs/${fileName}`;
+        window.open(filePath, '_blank');
 
         this.setState({
           loading: false