Commit f5307d6f authored by Caleb Weeks's avatar Caleb Weeks

Merge branch 'feature/mtc-1262' into 'master'

fix error handling and change overall download process

See merge request !4
parents 8cf27557 0aa77107
......@@ -2,6 +2,7 @@ var _=require("lodash");
var aws=require("aws-sdk");
var _url=require("url");
var fs = require("fs");
var md5File = require('md5-file');
var confPath = "siteConfig.environment.imageOptimizer.s3-backing";
var pluginId = "img-opt-s3-renditions";
var log=global.bl.logger.get(`blacklight.${pluginId}`)
......@@ -30,45 +31,61 @@ module.exports=function s3BackedRenditions(config, $){
/////////////////////////////////////////////////////////////////////////////
checkForRendition: (params, cb)=>{
var lastModifiedTime;
var localFile = fs.createWriteStream(params.renditionPathOnDisk);
var downloader = s3client.getObject({
Bucket: s3conf.bucket,
Key: params.renditionPath
}).on("httpHeaders", (statusCode, headers)=>{
lastModifiedTime = Date.parse(headers["last-modified"])
}).on("httpData", function(chunk) {
localFile.write(chunk);
}).on("httpDone", function(response) {
localFile.end();
if(lastModifiedTime){
fs.utimes(params.renditionPathOnDisk, lastModifiedTime, lastModifiedTime, ()=>{
log.info("Got s3 rendition", params.renditionPath);
cb(null, lastModifiedTime);
});
}
}).on("error", function(err){
var msg;
if(err && err.message){
if(err.statusCode===403 || err.code.match(/Forbidden/) || err.code.match(/AccessDenied/) || err.code.match(/Invalid/)){
msg = `Error downloading rendition from S3 bucket ${s3conf.bucket}. Please check config at: ${confPath}`;
emailErrorOccasionally(params.emailError, msg);
log.error(msg, err);
} else if(err.statsCode===404 || err.code.match(/NoSuch/) || err.code.match(/Not Found/)){
msg = `S3 cache miss, Generating new img-opt rendition. Bucket: ${s3conf.bucket} Path: ${params.renditionPath}`;
log.info(msg);
if (params && params.renditionPath && params.renditionPathOnDisk) {
var lastModifiedTime;
var renditionPath = params.renditionPath.replace(/^\/+/g, '');
var downloader = s3client.getObject({
Bucket: s3conf.bucket,
Key: renditionPath
}, function(err, data) {
if (err) {
var msg;
if(err.code.match(/Forbidden/) || err.code.match(/AccessDenied/) || err.code.match(/Invalid/)){
msg = `Error downloading rendition from S3 bucket ${s3conf.bucket}. Please check config at: ${confPath}`;
emailErrorOccasionally(params.emailError, msg);
log.error(msg, err);
} else if(err.code.match(/NoSuch/) || err.code.match(/Not Found/)){
msg = `S3 cache miss, Generating new img-opt rendition. Bucket: ${s3conf.bucket} Path: ${params.renditionPath}`;
log.info(msg);
} else {
msg = `Error downloading rendition from S3 bucket. ${err.message}`;
log.error(msg, err);
}
cb(msg);
} else {
msg = `Error downloading rendition from S3 bucket. ${err.message}`;
log.error(msg, err);
var headers = this.httpResponse.headers;
lastModifiedTime = Date.parse(headers['last-modified']);
fs.writeFile(params.renditionPathOnDisk, data.Body.toString('base64'), 'base64', (fileError) => {
var msg;
if (fileError) {
msg = "Error writing to filesystem";
log.error(msg, fileError);
cb(msg);
} else {
if (lastModifiedTime && headers['etag'].replace(/["]+/g, '') === md5File.sync(params.renditionPathOnDisk)) {
//fs.utimes(params.renditionPathOnDisk, lastModifiedTime, lastModifiedTime, ()=>{
log.info("Got s3 rendition", params.renditionPath);
cb(null, lastModifiedTime);
//});
} else {
fs.unlinkSync(params.renditionPathOnDisk);
msg = `Error downloading rendition from s3 Bucket: ${s3conf.bucket} Path: ${params.renditionPath}, hashes don't match`;
log.error(msg);
cb(msg);
}
}
});
}
}else{
msg=`Unknown S3 download error.`;
log.error(msg, err);
}
localFile.end();
});
} else {
var msg = "Error downloading rendition from S3, invalid parameters";
log.error(msg);
cb(msg);
}).send();
}
},
......
{
"name": "bl-plugin-img-opt-s3-renditions",
"version": "0.0.7",
"version": "0.0.8",
"description": "Store img-opt renditions in S3 so they can be shared amongst BL instances, which facilitates horizontal scaling.",
"main": "index.js",
"scripts": {
......@@ -9,7 +9,8 @@
"author": "",
"license": "ISC",
"dependencies": {
"aws-sdk": "^2.268.1",
"lodash": "^4.17.10"
"aws-sdk": "^2.437.0",
"lodash": "^4.17.10",
"md5-file": "^4.0.0"
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment