Ganti package untuk bypass Cloudflare

get-data-harian-emiten.py

@@ -1,4 +1,4 @@
-import urllib3, json
+import cfscrape, json
 import pandas as pd
 from time import sleep
 
@@ -15,7 +15,7 @@
 kode_lq45 = lq45['code'].values
 
 # http client
-http = urllib3.PoolManager()
+http = cfscrape.create_scraper()
 
 for code in kode_emiten:
 	# link
@@ -26,9 +26,10 @@
 	# request whenever failed
 	while True:
 		try:
-			result = http.request('GET', link)
-			result = http.request('GET', link)
-			result = json.loads(result.data.decode('utf-8'))
+			# send request
+			result = http.get(link).text
+			result = http.get(link).text
+			result = json.loads(result)
 
 			# success, we stop the while loop
 			break

get-data-tahunan-emiten.py

@@ -1,4 +1,4 @@
-import urllib3, json
+import cfscrape, json
 import pandas as pd
 from time import sleep
 
@@ -16,7 +16,7 @@
 lq45 = lq45['code'].values
 
 # http client
-http = urllib3.PoolManager()
+http = cfscrape.create_scraper()
 
 for code in emiten:
 	# link
@@ -27,8 +27,9 @@
 	# request whenever failed
 	while True:
 		try:
-			result = http.request('GET', link)
-			result = json.loads(result.data.decode('utf-8'))
+			# send request
+			result = http.get(link).text
+			result = json.loads(result)
 
 			# success, we brake the while loop
 			break

get-list-emiten.py

@@ -1,4 +1,4 @@
-import urllib3, json
+import cfscrape, json
 import pandas as pd
 
 # mulai dari 0
@@ -26,15 +26,15 @@
 lq45_listingBoard = []
 
 # http client
-http = urllib3.PoolManager()
+http = cfscrape.create_scraper()
 
 while True:
 	# buat link
 	link = f"https://idx.co.id/umbraco/Surface/StockData/GetSecuritiesStock?code=&sector=&board=&start={start}&length={length}"
 
 	# send request
-	result = http.request('GET', link)
-	result = json.loads(result.data.decode('utf-8'))
+	result = http.get(link).text
+	result = json.loads(result)
 
 	# result empty?
 	# kalo iya, berarti daftar

requirements.txt

@@ -1,3 +1,2 @@
-urllib3
-pandas
-kaggle
+cfscrape
+pandas