Skip to content

Commit

Permalink
Update notebooks
Browse files Browse the repository at this point in the history
  • Loading branch information
@Heizenburg
Heizenburg committed Oct 13, 2023
1 parent 1d1fa21 commit 2928141
Show file tree
Hide file tree
Showing 6 changed files with 1,250 additions and 210 deletions.
12 changes: 6 additions & 6 deletions notebooks/EST/devland.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
},
{
"cell_type": "code",
"execution_count": 29,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -117,7 +117,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -136,7 +136,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -157,7 +157,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -167,11 +167,11 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"with pd.ExcelWriter(\"C:\\\\Users\\\\tsello01\\\\Documents\\\\Data\\\\devland_files.xlsx\", engine='xlsxwriter') as writer:\n",
"with pd.ExcelWriter(\"C:\\\\Users\\\\tsello01\\\\Documents\\\\Data\\\\Devland Latest.xlsx\", engine='xlsxwriter') as writer:\n",
" all_files.to_excel(writer, sheet_name='All Files', index=False)\n",
" recent_files.to_excel(writer, sheet_name='Most Recent Files', index=False)"
]
Expand Down
112 changes: 65 additions & 47 deletions notebooks/EST/react.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -28,7 +28,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -57,7 +57,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {},
"outputs": [
{
Expand All @@ -66,7 +66,7 @@
"True"
]
},
"execution_count": 35,
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -78,9 +78,17 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"FTP Error: \n"
]
}
],
"source": [
"ftp_host = os.getenv('FTP_HOST')\n",
"ftp_user = os.getenv('FTP_USERNAME')\n",
Expand All @@ -97,7 +105,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -143,12 +151,12 @@
" else:\n",
" kept_files.append(os.path.basename(file))\n",
"\n",
" # Create a Pandas DataFrame of the deleted files\n",
" # Create a DataFrame of the deleted files\n",
" df_deleted = pd.DataFrame(deleted_files, columns=[\"Deleted Files\"])\n",
" df_deleted[\"Deleted Files\"] = df_deleted[\"Deleted Files\"].apply(lambda x: os.path.basename(x))\n",
" df_deleted = df_deleted.sort_values(by=[\"Deleted Files\"])\n",
"\n",
" # Create a Pandas DataFrame of the kept files with only the last two elements in the file path\n",
" # Create a DataFrame of the kept files with only the last two elements in the file path\n",
" df_kept = pd.DataFrame(kept_files, columns=[\"Kept Files\"])\n",
" df_kept[\"Kept Files\"] = df_kept[\"Kept Files\"].apply(lambda x: os.path.join(*os.path.split(x)[-2:]))\n",
" df_kept = df_kept.drop_duplicates()\n",
Expand All @@ -158,52 +166,62 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"( Deleted Files\n",
" 20 SaleAudit-2023-07-10.csv\n",
" 10 SaleAudit-2023-07-17.csv\n",
" 0 SaleAudit-2023-07-24.csv\n",
" 22 SaleAudit_A1 SUPERMARKET_2023-07-10.csv\n",
" 13 SaleAudit_A1 SUPERMARKET_2023-07-17.csv\n",
" 5 SaleAudit_A1 SUPERMARKET_2023-07-24.csv\n",
" 17 SaleAudit_Mogwase DC_2023-07-10.csv\n",
" 11 SaleAudit_Mogwase DC_2023-07-17.csv\n",
" 3 SaleAudit_Mogwase DC_2023-07-24.csv\n",
" 23 SaleAudit_Mogwase Grocery_2023-07-10.csv\n",
" 14 SaleAudit_Mogwase Grocery_2023-07-17.csv\n",
" 6 SaleAudit_Mogwase Grocery_2023-07-24.csv\n",
" 19 SaleAudit_Northam_2023-07-10.csv\n",
" 9 SaleAudit_Northam_2023-07-17.csv\n",
" 2 SaleAudit_Northam_2023-07-24.csv\n",
" 25 SaleAudit_Potchefstroom_2023-07-10.csv\n",
" 16 SaleAudit_Potchefstroom_2023-07-17.csv\n",
" 7 SaleAudit_Potchefstroom_2023-07-24.csv\n",
" 21 SaleAudit_Premjee LT_2023-07-10.csv\n",
" 12 SaleAudit_Premjee LT_2023-07-17.csv\n",
" 4 SaleAudit_Premjee LT_2023-07-24.csv\n",
" 18 SaleAudit_Premjee TZN_2023-07-10.csv\n",
" 8 SaleAudit_Premjee TZN_2023-07-17.csv\n",
" 1 SaleAudit_Premjee TZN_2023-07-24.csv\n",
" 24 SaleAudit_Zeerust_2023-07-10.csv\n",
" 15 SaleAudit_Zeerust_2023-07-17.csv,\n",
" Kept Files\n",
" 0 SaleAudit_Mogwase Grocery_2023-07-31.csv\n",
" 1 SaleAudit_Mogwase DC_2023-07-31.csv\n",
" 2 SaleAudit-2023-07-31.csv\n",
" 3 SaleAudit_Premjee TZN_2023-07-31.csv\n",
" 4 SaleAudit_Northam_2023-07-31.csv\n",
" 5 SaleAudit_A1 SUPERMARKET_2023-07-31.csv\n",
" 6 SaleAudit_Premjee LT_2023-07-31.csv\n",
" 7 SaleAudit_Potchefstroom_2023-07-31.csv\n",
" 8 SaleAudit_Zeerust_2023-07-24.csv)"
" 27 SaleAudit-2023-09-04.csv\n",
" 19 SaleAudit-2023-09-11.csv\n",
" 10 SaleAudit-2023-09-11.csv\n",
" 11 SaleAudit-2023-09-18.csv\n",
" 9 SaleAudit-2023-09-18.csv\n",
" 0 SaleAudit-2023-09-25.csv\n",
" 30 SaleAudit_A1 SUPERMARKET_2023-09-04.csv\n",
" 22 SaleAudit_A1 SUPERMARKET_2023-09-11.csv\n",
" 15 SaleAudit_A1 SUPERMARKET_2023-09-18.csv\n",
" 4 SaleAudit_A1 SUPERMARKET_2023-09-25.csv\n",
" 35 SaleAudit_Mogwase DC_2023-09-04.csv\n",
" 26 SaleAudit_Mogwase DC_2023-09-11.csv\n",
" 18 SaleAudit_Mogwase DC_2023-09-18.csv\n",
" 8 SaleAudit_Mogwase DC_2023-09-25.csv\n",
" 33 SaleAudit_Mogwase Grocery_2023-09-04.csv\n",
" 24 SaleAudit_Mogwase Grocery_2023-09-11.csv\n",
" 16 SaleAudit_Mogwase Grocery_2023-09-18.csv\n",
" 5 SaleAudit_Mogwase Grocery_2023-09-25.csv\n",
" 29 SaleAudit_Northam_2023-09-04.csv\n",
" 21 SaleAudit_Northam_2023-09-11.csv\n",
" 13 SaleAudit_Northam_2023-09-18.csv\n",
" 2 SaleAudit_Northam_2023-09-25.csv\n",
" 34 SaleAudit_Potchefstroom_2023-09-04.csv\n",
" 25 SaleAudit_Potchefstroom_2023-09-11.csv\n",
" 17 SaleAudit_Potchefstroom_2023-09-18.csv\n",
" 6 SaleAudit_Potchefstroom_2023-09-25.csv\n",
" 31 SaleAudit_Premjee LT_2023-09-04.csv\n",
" 23 SaleAudit_Premjee LT_2023-09-11.csv\n",
" 14 SaleAudit_Premjee LT_2023-09-18.csv\n",
" 3 SaleAudit_Premjee LT_2023-09-25.csv\n",
" 28 SaleAudit_Premjee TZN_2023-09-04.csv\n",
" 20 SaleAudit_Premjee TZN_2023-09-11.csv\n",
" 12 SaleAudit_Premjee TZN_2023-09-18.csv\n",
" 1 SaleAudit_Premjee TZN_2023-09-25.csv\n",
" 32 SaleAudit_Zeerust_2023-09-04.csv\n",
" 7 SaleAudit_Zeerust_2023-09-25.csv,\n",
" Kept Files\n",
" 0 SaleAudit_Northam_2023-10-02.csv\n",
" 1 SaleAudit_Premjee TZN_2023-10-02.csv\n",
" 3 SaleAudit_Premjee LT_2023-10-02.csv\n",
" 5 SaleAudit_A1 SUPERMARKET_2023-10-02.csv\n",
" 6 SaleAudit_Zeerust_2023-10-02.csv\n",
" 7 SaleAudit_Mogwase Grocery_2023-10-02.csv\n",
" 8 SaleAudit_Potchefstroom_2023-10-02.csv\n",
" 9 SaleAudit_Lichtenburg_2023-10-02.csv\n",
" 10 SaleAudit_Mogwase DC_2023-10-02.csv)"
]
},
"execution_count": 4,
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
Expand Down
101 changes: 58 additions & 43 deletions notebooks/Massmart/massmart_placement.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 25,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -16,22 +16,32 @@
},
{
"cell_type": "code",
"execution_count": 26,
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"source_folder = r'R:\\RawData\\Massmart Data'"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"parameters = [\n",
" {'target_dir': r'R:\\RawData\\Game\\ToUpload', 'keyword': 'MassDiscounters'},\n",
" {'target_dir': r'R:\\RawData\\Masscash\\\\Wholesale\\ToUpload', 'keyword': 'MassCashWholesale'},\n",
" {'target_dir': r'R:\\RawData\\Masscash\\Retail\\ToUpload', 'keyword': 'MassCashRetail'},\n",
" {'target_dir': r'R:\\RawData\\Makro Updated\\\\ToUpload', 'keyword': 'Makro'},\n",
" {'target_dir': r'R:\\RawData\\Game\\ToUpload', 'keyword': 'MassDiscounters'},\n",
" {'target_dir': r'R:\\RawData\\KitKat\\ToUpload', 'keyword': 'KitKat'},\n",
" {'target_dir': r'R:\\RawData\\Masscash\\Retail\\ToUpload', 'keyword': 'MassCashRetail'},\n",
" {'target_dir': r'R:\\RawData\\Masscash\\\\Wholesale\\ToUpload', 'keyword': 'MassCashWholesale'}\n",
" {'target_dir': r'R:\\RawData\\Massbuild\\ToUpload', 'keyword': 'Massbuild'}\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": 27,
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -57,65 +67,70 @@
},
{
"cell_type": "code",
"execution_count": 28,
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"source_folder = r'R:\\RawData\\Massmart Data'"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"last_active_time = time.time() \n",
"max_inactive_time = 60 * 2 "
"max_inactive_time = 60 * 5"
]
},
{
"cell_type": "code",
"execution_count": 30,
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" File \\\n",
"0 PandGSA_MassDiscounters_Delta_WE_20230730.txt \n",
"1 PandGSA_MassDiscounters_WE_20230806.txt \n",
"\n",
" New Location \n",
"0 R:\\RawData\\Game\\ToUpload\\PandGSA_MassDiscounte... \n",
"1 R:\\RawData\\Game\\ToUpload\\PandGSA_MassDiscounte... \n",
" File \\\n",
"0 PandG_Makro_Delta_WE_20230730.txt \n",
"1 PandG_Makro_WE_20230806.txt \n",
" File \\\n",
"0 GSKSA_KitKat_Delta_WE_20230910.txt \n",
"1 GSKSA_KitKat_Delta_WE_20230917.txt \n",
"2 GSKSA_KitKat_Delta_WE_20230924.txt \n",
"3 GSKSA_KitKat_WE_20231001.txt \n",
"4 RhodesSA_KitKat_Delta_WE_20230910.txt \n",
"5 RhodesSA_KitKat_Delta_WE_20230917.txt \n",
"6 RhodesSA_KitKat_Delta_WE_20230924.txt \n",
"7 RhodesSA_KitKat_WE_20231001.txt \n",
"8 TigerSA_KitKat_WE_20231001.txt \n",
"\n",
" New Location \n",
"0 R:\\RawData\\Makro Updated\\\\ToUpload\\PandG_Makro... \n",
"1 R:\\RawData\\Makro Updated\\\\ToUpload\\PandG_Makro... \n",
"No new files detected for the past 2 minutes. Terminating script.\n"
"0 R:\\RawData\\KitKat\\ToUpload\\GSKSA_KitKat_Delta_... \n",
"1 R:\\RawData\\KitKat\\ToUpload\\GSKSA_KitKat_Delta_... \n",
"2 R:\\RawData\\KitKat\\ToUpload\\GSKSA_KitKat_Delta_... \n",
"3 R:\\RawData\\KitKat\\ToUpload\\GSKSA_KitKat_WE_202... \n",
"4 R:\\RawData\\KitKat\\ToUpload\\RhodesSA_KitKat_Del... \n",
"5 R:\\RawData\\KitKat\\ToUpload\\RhodesSA_KitKat_Del... \n",
"6 R:\\RawData\\KitKat\\ToUpload\\RhodesSA_KitKat_Del... \n",
"7 R:\\RawData\\KitKat\\ToUpload\\RhodesSA_KitKat_WE_... \n",
"8 R:\\RawData\\KitKat\\ToUpload\\TigerSA_KitKat_WE_2... \n",
"No new files detected for the past 5 minutes. Terminating script.\n"
]
}
],
"source": [
"while True:\n",
" for params in parameters:\n",
" df = move_files_based_on_keyword(source_folder, params['target_dir'], params['keyword'])\n",
" if not df.empty:\n",
" last_active_time = time.time() \n",
" print(df)\n",
"def monitor_files(parameters, source_folder, max_inactive_time):\n",
" last_active_time = time.time()\n",
" \n",
" while True:\n",
" new_files_detected = False\n",
"\n",
" for params in parameters:\n",
" df = move_files_based_on_keyword(source_folder, params['target_dir'], params['keyword'])\n",
" if not df.empty:\n",
" last_active_time = time.time()\n",
" print(df)\n",
" new_files_detected = True\n",
"\n",
" inactive_time = time.time() - last_active_time\n",
"\n",
" inactive_time = time.time() - last_active_time # calculate the inactive time\n",
" if inactive_time >= max_inactive_time:\n",
" print(f\"No new files detected for the past {int(max_inactive_time/60)} minutes. Terminating script.\")\n",
" break\n",
"\n",
" if inactive_time >= max_inactive_time:\n",
" print(f\"No new files detected for the past {int(max_inactive_time/60)} minutes. Terminating script.\")\n",
" break\n",
" time.sleep(10)\n",
"\n",
" time.sleep(10) # wait for 10 seconds before checking again"
"monitor_files(parameters, source_folder, max_inactive_time)\n"
]
}
],
Expand Down
Loading

0 comments on commit 2928141

Please sign in to comment.