Refactoring code for full backtest.

IanLKaplan · Feb 8, 2023 · e544f1f · e544f1f
1 parent dec4964
commit e544f1f
Show file tree

Hide file tree

Showing 496 changed files with 1,688 additions and 25 deletions.
diff --git a/pairs_trading_backtest.py b/pairs_trading_backtest.py
diff --git a/read_market_data/MarketData.py b/read_market_data/MarketData.py
@@ -5,7 +5,7 @@
 
 from datetime import datetime, timedelta
 from utils.convert_date import convert_date
-from multiprocessing import Pool
+from multiprocessing import Pool, cpu_count
 
 
 class MarketData:
@@ -100,9 +100,10 @@ def read_data(self, symbol: str) -> pd.DataFrame:
 
     def get_close_data(self, stock_list: list) -> pd.DataFrame:
         # fetch the close data in parallel
+        n_cores = cpu_count()
         close_df = pd.DataFrame()
         assert len(stock_list) > 0
-        with Pool() as mp_pool:
+        with Pool(processes=(n_cores*2)) as mp_pool:
             close_list = mp_pool.map(self.read_data, stock_list)
         # close_list = list()
         # for sym in stock_list:

diff --git a/s_and_p_data/A.csv b/s_and_p_data/A.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,152.08
 2023-02-01,155.45
 2023-02-02,155.5
+2023-02-03,154.55
+2023-02-06,152.24
+2023-02-07,154.51
diff --git a/s_and_p_data/AAL.csv b/s_and_p_data/AAL.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,16.14
 2023-02-01,16.69
 2023-02-02,17.06
+2023-02-03,17.02
+2023-02-06,16.92
+2023-02-07,16.96
diff --git a/s_and_p_data/AAP.csv b/s_and_p_data/AAP.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,152.28
 2023-02-01,156.84
 2023-02-02,155.24
+2023-02-03,152.17
+2023-02-06,151.67
+2023-02-07,151.47
diff --git a/s_and_p_data/AAPL.csv b/s_and_p_data/AAPL.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,144.29
 2023-02-01,145.43
 2023-02-02,150.82
+2023-02-03,154.5
+2023-02-06,151.73
+2023-02-07,154.65
diff --git a/s_and_p_data/ABBV.csv b/s_and_p_data/ABBV.csv
@@ -2539,3 +2539,6 @@ Date,Close
 2023-01-31,147.75
 2023-02-01,146.6
 2023-02-02,144.84
+2023-02-03,145.2
+2023-02-06,145.02
+2023-02-07,145.14
diff --git a/s_and_p_data/ABC.csv b/s_and_p_data/ABC.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,168.96
 2023-02-01,161.05
 2023-02-02,158.47
+2023-02-03,156.73
+2023-02-06,154.49
+2023-02-07,156.79
diff --git a/s_and_p_data/ABT.csv b/s_and_p_data/ABT.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,110.55
 2023-02-01,111.82
 2023-02-02,112.25
+2023-02-03,111.79
+2023-02-06,110.75
+2023-02-07,111.99
diff --git a/s_and_p_data/ACN.csv b/s_and_p_data/ACN.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,279.05
 2023-02-01,283.6
 2023-02-02,294.1
+2023-02-03,289.22
+2023-02-06,284.48
+2023-02-07,287.82
diff --git a/s_and_p_data/ADBE.csv b/s_and_p_data/ADBE.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,370.34
 2023-02-01,383.92
 2023-02-02,392.23
+2023-02-03,379.33
+2023-02-06,375.23
+2023-02-07,383.82
diff --git a/s_and_p_data/ADI.csv b/s_and_p_data/ADI.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,171.47
 2023-02-01,176.59
 2023-02-02,179.3
+2023-02-03,178.82
+2023-02-06,177.55
+2023-02-07,181.02
diff --git a/s_and_p_data/ADM.csv b/s_and_p_data/ADM.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,82.85
 2023-02-01,83.37
 2023-02-02,82.31
+2023-02-03,82.28
+2023-02-06,82.56
+2023-02-07,82.66
diff --git a/s_and_p_data/ADP.csv b/s_and_p_data/ADP.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,225.81
 2023-02-01,226.73
 2023-02-02,228.99
+2023-02-03,225.31
+2023-02-06,223.47
+2023-02-07,226.77
diff --git a/s_and_p_data/ADSK.csv b/s_and_p_data/ADSK.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,215.16
 2023-02-01,222.19
 2023-02-02,228.22
+2023-02-03,223.66
+2023-02-06,218.34
+2023-02-07,223.65
diff --git a/s_and_p_data/AEE.csv b/s_and_p_data/AEE.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,86.87
 2023-02-01,87.29
 2023-02-02,86.73
+2023-02-03,85.3
+2023-02-06,85.9
+2023-02-07,85.99
diff --git a/s_and_p_data/AEP.csv b/s_and_p_data/AEP.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,93.96
 2023-02-01,94.65
 2023-02-02,94.55
+2023-02-03,92.11
+2023-02-06,92.5
+2023-02-07,92.28
diff --git a/s_and_p_data/AES.csv b/s_and_p_data/AES.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,27.41
 2023-02-01,26.8
 2023-02-02,26.52
+2023-02-03,26.24
+2023-02-06,26.21
+2023-02-07,26.42
diff --git a/s_and_p_data/AFL.csv b/s_and_p_data/AFL.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,73.5
 2023-02-01,73.28
 2023-02-02,68.9
+2023-02-03,69.39
+2023-02-06,69.58
+2023-02-07,69.95
diff --git a/s_and_p_data/AIG.csv b/s_and_p_data/AIG.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,63.22
 2023-02-01,61.23
 2023-02-02,59.03
+2023-02-03,58.75
+2023-02-06,59.96
+2023-02-07,60.61
diff --git a/s_and_p_data/AIZ.csv b/s_and_p_data/AIZ.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,132.59
 2023-02-01,132.25
 2023-02-02,133.58
+2023-02-03,133.63
+2023-02-06,133.13
+2023-02-07,135.65
diff --git a/s_and_p_data/AJG.csv b/s_and_p_data/AJG.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,195.72
 2023-02-01,196.28
 2023-02-02,194.68
+2023-02-03,193.49
+2023-02-06,194.55
+2023-02-07,197.47
diff --git a/s_and_p_data/AKAM.csv b/s_and_p_data/AKAM.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,88.95
 2023-02-01,89.58
 2023-02-02,91.28
+2023-02-03,89.33
+2023-02-06,87.49
+2023-02-07,88.16
diff --git a/s_and_p_data/ALB.csv b/s_and_p_data/ALB.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,281.45
 2023-02-01,285.9
 2023-02-02,290.15
+2023-02-03,287.23
+2023-02-06,276.81
+2023-02-07,279.34
diff --git a/s_and_p_data/ALGN.csv b/s_and_p_data/ALGN.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,269.73
 2023-02-01,282.53
 2023-02-02,359.88
+2023-02-03,343.1
+2023-02-06,342.46
+2023-02-07,345.07
diff --git a/s_and_p_data/ALK.csv b/s_and_p_data/ALK.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,51.34
 2023-02-01,51.84
 2023-02-02,53.58
+2023-02-03,53.47
+2023-02-06,52.43
+2023-02-07,51.8
diff --git a/s_and_p_data/ALL.csv b/s_and_p_data/ALL.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,128.47
 2023-02-01,128.16
 2023-02-02,131.89
+2023-02-03,131.33
+2023-02-06,134.98
+2023-02-07,136.72
diff --git a/s_and_p_data/ALLE.csv b/s_and_p_data/ALLE.csv
@@ -2317,3 +2317,6 @@ Date,Close
 2023-01-31,117.55
 2023-02-01,120.05
 2023-02-02,121.74
+2023-02-03,122.93
+2023-02-06,120.87
+2023-02-07,120.26
diff --git a/s_and_p_data/AMAT.csv b/s_and_p_data/AMAT.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,111.49
 2023-02-01,119.01
 2023-02-02,124.26
+2023-02-03,119.98
+2023-02-06,117.69
+2023-02-07,120.52
diff --git a/s_and_p_data/AMCR.csv b/s_and_p_data/AMCR.csv
@@ -2697,3 +2697,6 @@ Date,Close
 2023-01-31,12.06
 2023-02-01,12.07
 2023-02-02,12.2
+2023-02-03,12.03
+2023-02-06,11.93
+2023-02-07,11.89
diff --git a/s_and_p_data/AMD.csv b/s_and_p_data/AMD.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,75.15
 2023-02-01,84.64
 2023-02-02,88.31
+2023-02-03,86.09
+2023-02-06,83.68
+2023-02-07,85.91
diff --git a/s_and_p_data/AME.csv b/s_and_p_data/AME.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,144.92
 2023-02-01,145.15
 2023-02-02,144.52
+2023-02-03,146.38
+2023-02-06,147.09
+2023-02-07,146.74
diff --git a/s_and_p_data/AMGN.csv b/s_and_p_data/AMGN.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,252.4
 2023-02-01,246.41
 2023-02-02,246.53
+2023-02-03,245.17
+2023-02-06,243.85
+2023-02-07,244.0
diff --git a/s_and_p_data/AMP.csv b/s_and_p_data/AMP.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,350.12
 2023-02-01,354.53
 2023-02-02,347.71
+2023-02-03,354.1
+2023-02-06,351.03
+2023-02-07,353.33
diff --git a/s_and_p_data/AMT.csv b/s_and_p_data/AMT.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,223.39
 2023-02-01,227.14
 2023-02-02,227.39
+2023-02-03,222.06
+2023-02-06,220.57
+2023-02-07,219.47
diff --git a/s_and_p_data/AMZN.csv b/s_and_p_data/AMZN.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,103.13
 2023-02-01,105.15
 2023-02-02,112.91
+2023-02-03,103.39
+2023-02-06,102.18
+2023-02-07,102.11
diff --git a/s_and_p_data/ANET.csv b/s_and_p_data/ANET.csv
@@ -2180,3 +2180,6 @@ Date,Close
 2023-01-31,126.02
 2023-02-01,130.52
 2023-02-02,133.44
+2023-02-03,131.96
+2023-02-06,129.69
+2023-02-07,134.91
diff --git a/s_and_p_data/ANSS.csv b/s_and_p_data/ANSS.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,266.36
 2023-02-01,274.39
 2023-02-02,282.5
+2023-02-03,275.23
+2023-02-06,272.43
+2023-02-07,277.09
diff --git a/s_and_p_data/AON.csv b/s_and_p_data/AON.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,318.68
 2023-02-01,319.38
 2023-02-02,323.0
+2023-02-03,313.64
+2023-02-06,317.73
+2023-02-07,319.8
diff --git a/s_and_p_data/AOS.csv b/s_and_p_data/AOS.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,67.7
 2023-02-01,69.96
 2023-02-02,70.4
+2023-02-03,69.05
+2023-02-06,67.49
+2023-02-07,66.63
diff --git a/s_and_p_data/APA.csv b/s_and_p_data/APA.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,44.33
 2023-02-01,43.4
 2023-02-02,42.36
+2023-02-03,42.04
+2023-02-06,42.02
+2023-02-07,43.69
diff --git a/s_and_p_data/APD.csv b/s_and_p_data/APD.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,320.51
 2023-02-01,318.13
 2023-02-02,295.5
+2023-02-03,285.02
+2023-02-06,283.03
+2023-02-07,285.07
diff --git a/s_and_p_data/APH.csv b/s_and_p_data/APH.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,79.77
 2023-02-01,81.14
 2023-02-02,82.09
+2023-02-03,81.62
+2023-02-06,81.8
+2023-02-07,81.35
diff --git a/s_and_p_data/APTV.csv b/s_and_p_data/APTV.csv
@@ -2819,3 +2819,6 @@ Date,Close
 2023-01-31,113.09
 2023-02-01,114.75
 2023-02-02,118.68
+2023-02-03,115.73
+2023-02-06,114.95
+2023-02-07,115.91
diff --git a/s_and_p_data/ARE.csv b/s_and_p_data/ARE.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,160.74
 2023-02-01,163.27
 2023-02-02,170.82
+2023-02-03,167.79
+2023-02-06,167.26
+2023-02-07,166.83
diff --git a/s_and_p_data/ATO.csv b/s_and_p_data/ATO.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,117.54
 2023-02-01,118.97
 2023-02-02,119.55
+2023-02-03,117.19
+2023-02-06,117.64
+2023-02-07,116.9
diff --git a/s_and_p_data/ATVI.csv b/s_and_p_data/ATVI.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,76.57
 2023-02-01,76.7
 2023-02-02,77.11
+2023-02-03,75.24
+2023-02-06,71.58
+2023-02-07,75.6
diff --git a/s_and_p_data/AVB.csv b/s_and_p_data/AVB.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,177.44
 2023-02-01,178.02
 2023-02-02,184.5
+2023-02-03,180.25
+2023-02-06,178.79
+2023-02-07,179.71
diff --git a/s_and_p_data/AVGO.csv b/s_and_p_data/AVGO.csv
@@ -3396,3 +3396,6 @@ Date,Close
 2023-01-31,585.01
 2023-02-01,602.75
 2023-02-02,605.77
+2023-02-03,597.62
+2023-02-06,601.3
+2023-02-07,614.45
diff --git a/s_and_p_data/AVY.csv b/s_and_p_data/AVY.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,189.44
 2023-02-01,191.66
 2023-02-02,188.56
+2023-02-03,181.2
+2023-02-06,178.54
+2023-02-07,183.6
diff --git a/s_and_p_data/AWK.csv b/s_and_p_data/AWK.csv
@@ -3721,3 +3721,6 @@ Date,Close
 2023-01-31,156.49
 2023-02-01,158.62
 2023-02-02,160.6
+2023-02-03,156.61
+2023-02-06,156.41
+2023-02-07,155.47
diff --git a/s_and_p_data/AXP.csv b/s_and_p_data/AXP.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,174.93
 2023-02-01,173.61
 2023-02-02,173.13
+2023-02-03,178.86
+2023-02-06,176.48
+2023-02-07,178.7
diff --git a/s_and_p_data/AZO.csv b/s_and_p_data/AZO.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,2438.85
 2023-02-01,2495.68
 2023-02-02,2443.55
+2023-02-03,2398.51
+2023-02-06,2425.88
+2023-02-07,2419.81
diff --git a/s_and_p_data/BA.csv b/s_and_p_data/BA.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,213.0
 2023-02-01,214.75
 2023-02-02,209.34
+2023-02-03,206.01
+2023-02-06,206.81
+2023-02-07,214.76
diff --git a/s_and_p_data/BAC.csv b/s_and_p_data/BAC.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,35.48
 2023-02-01,35.92
 2023-02-02,36.13
+2023-02-03,36.43
+2023-02-06,36.37
+2023-02-07,36.77
diff --git a/s_and_p_data/BALL.csv b/s_and_p_data/BALL.csv
@@ -4049,3 +4049,6 @@ Date,Close
 2023-01-31,58.24
 2023-02-01,58.44
 2023-02-02,60.89
+2023-02-03,58.97
+2023-02-06,58.69
+2023-02-07,57.6