fix paths

lib/constants.js

@@ -20,7 +20,7 @@ const SHEET_DEATHS_AND_RECOVERIES = "o3biev0";
 const SHEET_SOURCES_LIST = "obndi9r"
 
 
-const DIR = "./";
+const DIR = "./tmp/";
 
 const FILE_RAW_DATA = "/raw_data.json";
 const FILE_DATA = "/data.json";

lib/index.js

@@ -24,7 +24,7 @@ const fetchData = async ({sheet: sheet, tabs: tabs}) => {
 const writeData = async ({ file: file, data: data }) => {
     const fileContent = JSON.stringify(sortObjByKey(data),null,"\t");
     if (!fs.existsSync(DIR)) {
-        fs.mkdirSync(DIR);
+        fs.mkdirSync(DIR, { recursive: true });
     }
     return fs.writeFileSync(DIR + file, fileContent);
 };

pushthejson

@@ -5,30 +5,38 @@ set -eu
 repo_uri="https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
 remote_name="origin"
 main_branch="master"
-target_branch="gh-pages"
+gh_pages_branch="gh-pages"
 
 cd "$GITHUB_WORKSPACE"
 
 git config user.name "$GITHUB_ACTOR"
 git config user.email "${GITHUB_ACTOR}@bots.github.com"
 
-# git checkout "$target_branch"
+
 # git rebase "${remote_name}/${main_branch}"
 
+git checkout "$gh_pages_branch"
+
+mkdir tmp
+mkdir tmp/resources
+mkdir tmp/v2
+mkdir tmp/updatelog
+
+cp ./data.json .tmp/data_prev.json
+
+git checkout "$main_branch"
 
-cp ./data.json ./data_prev.json
+node src/sheets-to-csv.js
+node src/sheet-to-json_generic.js
+node src/raw_data-to-state_district_wise_data.js
+node src/generate_activity_log.js ${BOT_TOKEN}
+node src/states_daily_to_csv.js
 
-# node sheets-to-csv.js
-node sheet-to-json_generic.js
-node raw_data-to-state_district_wise_data.js
-node generate_activity_log.js ${BOT_TOKEN}
-node states_daily_to_csv.js
-touch test.json
 
 
 rm ./data_prev.json
 
-git checkout "$target_branch"
+git checkout "$gh_pages_branch"
 
 git add updatelog/ csv/ *.json 
 set +e  # Grep succeeds with nonzero exit codes to show results.
@@ -38,7 +46,7 @@ then
     set -e
     git commit -am "data updated on - $(date)"
     git remote set-url "$remote_name" "$repo_uri" # includes access token
-    git push --force-with-lease "$remote_name" "$target_branch"
+    git push --force-with-lease "$remote_name" "$gh_pages_branch"
 else
     set -e
     echo "No changes since last run"

src/generate_activity_log.js

@@ -1,10 +1,10 @@
 const fs = require('fs');
 const fetch = require('node-fetch');
 const moment = require('moment-timezone');
-const data = require('./data.json');
-const data_prev = require('./data_prev.json');
+const data = require('../tmp/data.json');
+const data_prev = require('../tmp/data_prev.json');
 
-update_log_file = './updatelog/log.json';
+update_log_file = './tmp/updatelog/log.json';
 var update_log = require(update_log_file);
 BOT_TOKEN = process.argv[2];
 

src/raw_data-to-state_district_wise_data.js

@@ -1,69 +1,66 @@
 const fs = require('fs');
 const moment = require("moment");
-const rawData = require('./raw_data');
+const rawData = require('../tmp/raw_data.json');
 
 console.log('Starting district wise data processing');
-try {
-  const StateDistrictWiseData = rawData.raw_data.reduce((acc, row) => {
-    const todaysdate = moment().utcOffset(330); //set moment instance to IST timezone and fetch currect IST time
-    const isToday = moment(todaysdate).add(-420, "m").isSame(moment(row.dateannounced + " 00:00 +0530", "DD-MM-YYYY HH:mm Z"), "day"); //Subtract 7 hours from IST time to persist values upto 7 AM and create moment instance from row.dateannounced in the IST Timezone
-    let stateName = row.detectedstate;
-      if(!stateName) {
-        return acc;
-        stateName = 'Unknown';
-      }
-    if(!acc[stateName]) {
-      acc[stateName] = {districtData: {}};
-    }
-    let districtName = row.detecteddistrict;
-      if(!districtName) {
-        districtName = 'Unknown';
-      }
-    if(!acc[stateName].districtData[districtName]) {
-
-      acc[stateName].districtData[districtName] = {
-//         active: 0,
-        confirmed: 0,
-//         deaths: 0,
-        lastupdatedtime: "",
-//         recovered: 0,
-        delta: {
-          confirmed: 0
-        }
-      };
-    }
-    const currentDistrict = acc[stateName].districtData[districtName];
-
-    currentDistrict.confirmed++;
-    if (isToday) {
-      currentDistrict.delta.confirmed++;
-    }
-//     if(row.currentstatus === 'Hospitalized') {
-//       currentDistrict.active++;
-//     } else if(row.currentstatus === 'Deceased') {
-//       currentDistrict.deaths++;
-//     } else if(row.currentstatus === 'Recovered') {
-//       currentDistrict.recovered++;
-//     }
 
+const StateDistrictWiseData = rawData.raw_data.reduce((acc, row) => {
+  const todaysdate = moment().utcOffset(330); //set moment instance to IST timezone and fetch currect IST time
+  const isToday = moment(todaysdate).add(-420, "m").isSame(moment(row.dateannounced + " 00:00 +0530", "DD-MM-YYYY HH:mm Z"), "day"); //Subtract 7 hours from IST time to persist values upto 7 AM and create moment instance from row.dateannounced in the IST Timezone
+  let stateName = row.detectedstate;
+  if (!stateName) {
     return acc;
-
-  }, {});
+    stateName = 'Unknown';
+  }
+  if (!acc[stateName]) {
+    acc[stateName] = { districtData: {} };
+  }
+  let districtName = row.detecteddistrict;
+  if (!districtName) {
+    districtName = 'Unknown';
+  }
+  if (!acc[stateName].districtData[districtName]) {
+
+    acc[stateName].districtData[districtName] = {
+      //         active: 0,
+      confirmed: 0,
+      //         deaths: 0,
+      lastupdatedtime: "",
+      //         recovered: 0,
+      delta: {
+        confirmed: 0
+      }
+    };
+  }
+  const currentDistrict = acc[stateName].districtData[districtName];
+
+  currentDistrict.confirmed++;
+  if (isToday) {
+    currentDistrict.delta.confirmed++;
+  }
+  //     if(row.currentstatus === 'Hospitalized') {
+  //       currentDistrict.active++;
+  //     } else if(row.currentstatus === 'Deceased') {
+  //       currentDistrict.deaths++;
+  //     } else if(row.currentstatus === 'Recovered') {
+  //       currentDistrict.recovered++;
+  //     }
+
+  return acc;
+
+}, {});
 
-  let stateDistrictWiseDataV2 = Object.keys(StateDistrictWiseData).map(state => {
-    let districtData = StateDistrictWiseData[state].districtData;
-    return {
-      state,
-      districtData: Object.keys(districtData).map(district => {
-        return { district, ...districtData[district] };
-      })
-    }
-  });
+let stateDistrictWiseDataV2 = Object.keys(StateDistrictWiseData).map(state => {
+  let districtData = StateDistrictWiseData[state].districtData;
+  return {
+    state,
+    districtData: Object.keys(districtData).map(district => {
+      return { district, ...districtData[district] };
+    })
+  }
+});
 
-  fs.writeFileSync('state_district_wise.json', JSON.stringify(StateDistrictWiseData, null, 2));
-  fs.writeFileSync('./v2/state_district_wise.json', JSON.stringify(stateDistrictWiseDataV2, null, 2));
-  console.log('Starting district wise data processing ...done');
-} catch(err) {
-  console.log('Error processing district wise data', err);
-}
+fs.writeFileSync('./tmp/state_district_wise.json', JSON.stringify(StateDistrictWiseData, null, 2));
+fs.writeFileSync('./tmp/v2/state_district_wise.json', JSON.stringify(stateDistrictWiseDataV2, null, 2));
+console.log('Starting district wise data processing ...done');
 

src/sheet-to-json_generic.js

@@ -1,5 +1,5 @@
-const { task, fetchData, writeData } = require("./lib");
-const c = require("./lib/constants");
+const { task, fetchData, writeData } = require("../lib");
+const c = require("../lib/constants");
 
 
 (async function main() {

src/sheets-to-csv.js

@@ -2,9 +2,13 @@ const fetch = require('node-fetch');
 const moment = require('moment-timezone');
 var fs = require('fs');
 
-var dir = './csv/';
+// var dir = __dirname + '/../tmp/csv/';
+// if (!fs.existsSync(dir)) {
+//     fs.mkdirSync(dir, { recursive: true });
+// }
+var dir = './tmp/csv/';
 if (!fs.existsSync(dir)) {
-    fs.mkdirSync(dir);
+    fs.mkdirSync(dir, { recursive: true });
 }
 
 const now = moment().unix()
@@ -35,21 +39,21 @@ const all_sheets = [
 ];
 
 all_sheets.forEach(element => {
-    console.log("Reading: "+element[0]);
+    console.log("Reading: " + element[0]);
     var temp_url = "https://docs.google.com/spreadsheets/d/e/" + PUBLISHED_SHEET_ID + "/pub?gid=" + element[1] + "&single=false&output=csv";
     console.log(temp_url);
     url = encodeURI(temp_url);
     let settings = { method: "Get" };
     fetch(url, settings).then(res => res.text())
         .then(csv => {
-            if(csv.includes("</html>")){
+            if (csv.includes("</html>")) {
                 console.error("probably not csv!");
                 process.exit(1);
                 return;
-            }else{
-                fs.writeFileSync(today_dir + "/"+element[0]+".csv", csv);
-                fs.writeFileSync(latest_dir + "/"+element[0]+".csv", csv);
-                console.log("Write completed: "+element[0]);
+            } else {
+                fs.writeFileSync(today_dir + "/" + element[0] + ".csv", csv);
+                fs.writeFileSync(latest_dir + "/" + element[0] + ".csv", csv);
+                console.log("Write completed: " + element[0]);
             }
         });
 });

src/states_daily_to_csv.js

@@ -1,4 +1,4 @@
-const daily = require("./states_daily.json");
+const daily = require("../tmp/states_daily.json");
 const fs = require('fs');
 
 var confirmed_csv = "date,TT,";
@@ -47,7 +47,11 @@ daily.states_daily.forEach(element => {
 // console.log(confirmed_csv);
 // console.log(recovered_csv);
 // console.log(deceased_csv);
+const csv_path = "tmp/states_daily_csv/";
+if (!fs.existsSync(csv_path)) {
+    fs.mkdirSync(csv_path, { recursive: true });
+}
 
-fs.writeFileSync('states_daily_csv/confirmed.csv', confirmed_csv);
-fs.writeFileSync('states_daily_csv/recovered.csv', recovered_csv);
-fs.writeFileSync('states_daily_csv/deceased.csv', deceased_csv);
+fs.writeFileSync(csv_path+'confirmed.csv', confirmed_csv);
+fs.writeFileSync(csv_path+'recovered.csv', recovered_csv);
+fs.writeFileSync(csv_path+'deceased.csv', deceased_csv);