Skip to content

Commit

Permalink
Merge pull request #112 from bcgov/refactor
Browse files Browse the repository at this point in the history 
fixed file cache
  • Loading branch information
@michaeltangbcgov
michaeltangbcgov authored Nov 21, 2023
2 parents 33da054 + aeae072 commit ff57439
Show file tree
Hide file tree
Showing 10 changed files with 106 additions and 183 deletions.
3 changes: 2 additions & 1 deletion backend/src/components/cache.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ const NodeCache = require("node-cache");
const listCache = new NodeCache({ stdTTL: 21600 });
const schoolCache = new NodeCache({ stdTTL: 21600 });
const codeCache = new NodeCache({ stdTTL: 21600 });
const fileCache = new NodeCache({ stdTTL: 21600 });

module.exports = {
listCache,schoolCache,codeCache
listCache,schoolCache,codeCache, fileCache
};
30 changes: 2 additions & 28 deletions backend/src/components/utils.js
View file
Original file line number Diff line number Diff line change
@@ -1,29 +1,8 @@
const ALLOWED_FILENAMES = new Set([
'trans',
'independent-authority-rep',
'indigenous',
'continuing-custody-order',
'distributed-learning',
'online-learning-contact',
'early-learning-child-care',
'planning-officer',
'early-learning',
'facilities',
'financial',
'french',
'international-education',
'literacy',
'myed-bc',
'inclusive-education',
'transportation',
'superintendent',
'chairperson',
'secretary-treasurer',
'executive-admin-assistant',

'districtcontacts',
'districtmailing',
'publicschoolcontacts',
'offshoreschoolcontacts',
'independentschoolcontacts',
'allschoolcontacts',
'allschoolmailing',
Expand Down Expand Up @@ -101,7 +80,6 @@ function removeFieldsByCriteria(inputData, criteria) {
function appendMailingAddressDetailsAndRemoveAddresses(data) {
if (data && data.addresses && data.addresses.length > 0) {
const physicalAddress = data.addresses.find(address => address.addressTypeCode === 'PHYSICAL');

if (physicalAddress) {
// Extract specific name-value pairs from the mailing address
const { addressLine1, addressLine2, city, postal, provinceCode, countryCode } = physicalAddress;
Expand Down Expand Up @@ -131,7 +109,6 @@ function appendMailingAddressDetailsAndRemoveAddresses(data) {
data.mailingCountryCode = countryCode;

// Remove the "addresses" property

}
delete data.addresses;
delete data.contacts;
Expand Down Expand Up @@ -216,8 +193,8 @@ function addDistrictLabels(jsonData, districtList) {
});
return reorderedObject;
}
function normalizeJsonObject(sourceArray, referenceArray, matchKey, condition, includeFields) {

function normalizeJsonObject(sourceArray, referenceArray, matchKey, condition, includeFields) {
return sourceArray.map((item) => {
const matchingItem = referenceArray.find(
(info) => info[matchKey] === item[matchKey] && (!condition || condition(info))
Expand All @@ -227,13 +204,10 @@ function addDistrictLabels(jsonData, districtList) {
...item,
...includeFields.reduce((result, field) => {
result[matchKey + "_" + field] = matchingItem[field];


return result;
}, {}),
};
}

return item;
});
}
Expand Down
72 changes: 30 additions & 42 deletions backend/src/routes/district-router.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ const axios = require("axios");
const fs = require("fs");
const path = require("path");
const { checkToken } = require("../components/auth");
const { listCache } = require("../components/cache");
const { listCache} = require("../components/cache");
const {getArrayofPubliclyAvailableCodes,filterRemoveByField, filterByExpiryDate, getArrayofNonPubliclyAvailableCodes, filterByField,appendMailingAddressDetailsAndRemoveAddresses, rearrangeAndRelabelObjectProperties, addDistrictLabels, normalizeJsonObject, sortJSONByDistrictNumber, removeFieldsByCriteria, filterByPubliclyAvailableCodes} = require("../components/utils.js")

//Batch Routes
Expand All @@ -16,45 +16,7 @@ router.get("/all-mailing", checkToken, getAllDistrictMailing);
router.get("/:id", checkToken, getDistrict);


async function removeItemsFromDistrictDataResponse(response, itemsToRemove) {
if (response && response.data) {
const newData = { ...response.data };

if (itemsToRemove && Array.isArray(itemsToRemove)) {
itemsToRemove.forEach((item) => {
if (newData[item]) {
delete newData[item];
}
});
}

response.data = newData;
}
}

async function getDistrictCodes(req) {
if (!listCache.has("districtCodesList")) {
const url = `${config.get(
"server:instituteAPIURL"
)}/institute/district-contact-type-codes`; // Update the URL according to your API endpoint
try {
const response = await axios.get(url, {
headers: { Authorization: `Bearer ${req.accessToken}` },
});
const districtCodeList = response.data;
listCache.set("districtCodesList", districtCodeList);
return districtCodeList;
} catch (e) {
log.error(
"getDistrictList Error",
e.response ? e.response.status : e.message
);
}
} else {
const districtCodeList = await listCache.get("districtCodesList");
return districtCodeList;
}
}
function getNonPublicContactTypeCodes(contactTypes) {
const nonPublicContactTypeCodes = [];

Expand Down Expand Up @@ -106,10 +68,35 @@ function removeContacts(districtDataResponse, nonPublicContactTypeCodes) {

return updatedDistrictData;
}
async function getDistrictCodes(req) {
if (!listCache.has("districtCodesList")) {
const url = `${config.get(
"server:instituteAPIURL"
)}/institute/district-contact-type-codes`; // Update the URL according to your API endpoint
try {
const response = await axios.get(url, {
headers: { Authorization: `Bearer ${req.accessToken}` },
});
const districtCodeList = response.data;
listCache.set("districtCodesList", districtCodeList);
return districtCodeList;
} catch (e) {
log.error(
"getDistrictList Error",
e.response ? e.response.status : e.message
);
}
} else {
const districtCodeList = await listCache.get("districtCodesList");
return districtCodeList;
}
}
async function getAllDistrictContacts(req, res) {

const districtList = await listCache.get("districtlist")
const contactTypeCodes= await listCache.get("codesList")
const districtAddresses = await listCache.get("districtAddresses")

let currentDate = new Date().toISOString().substring(0, 19)
const params = [
{
Expand Down Expand Up @@ -194,9 +181,7 @@ async function getAllDistrictContacts(req, res) {
array[index] = rearrangedElement;
});
let sortedData = sortJSONByDistrictNumber(filteredData)
const validDistricts = filterRemoveByField(sortedData,"District Number", ["098","102","103"])

res.json(validDistricts);
res.json(sortedData);

} catch (e) {
log.error("getData Error", e.response ? e.response.status : e.message);
Expand Down Expand Up @@ -330,6 +315,9 @@ async function getDistrict(req, res) {
const contactTypeCodes = await getDistrictCodes(req);
const districtContactCodeTypes = await listCache.get("codesList")
const nonPublicContactTypeCodes = getNonPublicContactTypeCodes(contactTypeCodes);
console.log("REVIEW")
console.log(nonPublicContactTypeCodes)

const districtDataPublic = removeContacts(
districtDataResponse.data,
nonPublicContactTypeCodes
Expand Down
68 changes: 7 additions & 61 deletions backend/src/routes/download-router.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,10 @@ const jsonExport = require('jsonexport');
const fs = require('fs');
const path = require('path');
const {isSafeFilePath} = require("../components/utils")
const { listCache } = require("../components/cache");


const { listCache, fileCache } = require("../components/cache");
const FILE_STORAGE_DIR = path.join(__dirname, '../..', 'public');

router.get('/csv/*', checkToken, getDownload, addDistrictLabels, createCSVFile, getCSVDownload);
router.get('/csv/*', checkToken, getDownload, createCSVFile, getCSVDownload);
router.get('/clear-files/:token', clearCSVFiles);

async function clearCSVFiles(req, res) {
Expand All @@ -32,31 +30,26 @@ async function clearCSVFiles(req, res) {
// Delete each file
fs.unlinkSync(filePath);
});

res.status(200).send('All files in the directory deleted successfully.');
} catch (error) {
console.error(error);
res.status(500).send('Internal Server Error');
}
}


async function createCSVFile(req,res, next){
try {


try {
jsonExport(req.jsonData, async function(err, csv){
if (err) return console.error(err);
await writeFileAsync(filePath, csv, 'binary');
next();
});


} catch (error) {
console.error("Error:", error);
res.status(500).send("Internal server error- Write File Sync issue");
}
}

async function writeFileAsync(filePath, data, encoding) {
return new Promise((resolve, reject) => {
fs.writeFile(filePath, data, encoding, (error) => {
Expand All @@ -69,68 +62,22 @@ async function writeFileAsync(filePath, data, encoding) {
});
}

async function addDistrictLabels(req, res, next) {
try {
let districtList = [];
if (listCache.has("districtlist")) {
districtList= listCache.get("districtlist");
} else {
try {
const path = "/api/v1/institute/district/list"
const url = `${req.protocol}://${req.hostname}:8080${path}`;

const response = await axios.get(url, { headers: { Authorization: `Bearer ${req.accessToken}` } });
const districts = response.data;
listCache.set("districtlist", districts);
districtList = districts

} catch (error) {
// Handle errors during the API request
throw new Error('Error fetching districts: ' + error.message);
}
}

if (req.jsonData && Array.isArray(req.jsonData)) {
req.jsonData.forEach(dataItem => {
const district = districtList.find(item => item.districtId === dataItem.districtId);
if (district) {
dataItem.districtNumber = district.districtNumber;
dataItem.displayName = district.displayName;
}
});
}

next();
} catch (error) {
// Handle errors here
console.error(error);
res.status(500).send('An error occurred.');
}
};


async function getDownload(req, res,next){

const filepath = req.query.filepath;

if (!filepath) {
if (!filepath) {s
return res.status(400).send("Missing 'filepath' parameter");
}else{
if (!isSafeFilePath(filepath)) {
return res.status(400).send("Invalid 'filepath' parameter");
}
}
filePath = path.join(FILE_STORAGE_DIR, `${filepath}.csv`);

if (fs.existsSync(filePath)) {
if(fileCache.has(filepath)){
res.setHeader('Content-Disposition', `attachment; filename="${filepath}.csv"`);
return res.sendFile(filePath);
}else{
try {


const path = req.url.replace('/csv', ''); // Modify the URL path as needed

const url =`${config.get("server:backend")}/v1${path}`
const response = await axios.get(url, { headers: { Authorization: `Bearer ${req.accessToken}` } });
// Attach the fetched data to the request object
Expand All @@ -139,7 +86,7 @@ async function getDownload(req, res,next){
}else{
req.jsonData = response.data;
}

fileCache.set(filepath, req.jsonData)
next(); // Call the next middleware
} catch (error) {
console.error("Error:", error);
Expand All @@ -158,7 +105,6 @@ async function getCSVDownload(req, res) {
return res.status(400).send("Invalid 'filepath' parameter");
}
}

const filePath = path.join(FILE_STORAGE_DIR, `${filepath}.csv`);
res.sendFile(filePath);
} catch (error) {
Expand Down
3 changes: 3 additions & 0 deletions backend/src/routes/school-router.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@ async function getAllSchools(req, res) {
const contactTypeCodes= await listCache.get("codesList")
let params = [];


if (await !schoolCache.has("openschoollist" + schoolCategory)) {
console.log("BUILDING CACHE FOR " + "openschoollist" + schoolCategory)
let currentDate = new Date().toISOString().substring(0, 19)
params = [

Expand Down Expand Up @@ -225,6 +227,7 @@ async function getAllSchools(req, res) {
);
});
} else {
console.log("USING CACHE" + " openschoollist" + schoolCategory)
const openSchoolList = await schoolCache.get("openschoollist" + schoolCategory);
res.json(openSchoolList);
}
Expand Down
Loading

0 comments on commit ff57439

Please sign in to comment.