S3 Upload Restarts and never completes
See original GitHub issueI noticed anytime I try to upload to S3, teh upload progress restarts. I’m using partial upload with the header Upload-Concat: "partial"
. When I use the FileStore, it works but Amazon S3 buckets gives this issue.
server.js
const tus = require('tus-node-server');
const express = require('express')
const path = require('path')
const fs = require('fs')
const spdy = require('spdy')
const cors = require('cors')
const uniqid = require('uniqid')
const mimeToExt = require('mime-to-extensions')
const fileExtension = require('file-extension');
const options = {
key: fs.readFileSync(__dirname + '/server.key'),
cert: fs.readFileSync(__dirname + '/server.crt')
}
const parseMetadataString = (metadata_string) => {
const kv_pair_list = metadata_string.split(',');
return kv_pair_list.reduce((metadata, kv_pair) => {
const [key, base64_value] = kv_pair.split(' ');
metadata[key] = {
encoded: base64_value,
decoded: Buffer.from(base64_value, 'base64').toString('ascii'),
};
return metadata;
}, {});
}
const fileNameFromUrl = (req) => {
let metadata = parseMetadataString( req.headers['upload-metadata'] )
return `${uniqid()}.${fileExtension( metadata.filetype.decoded )}`;
}
const server = new tus.Server();
server.datastore = new tus.S3Store({
path: '/files',
bucket: process.env.S3_BUCKET,
accessKeyId: process.env.AWS_ACCESS,
secretAccessKey: process.env.AWS_SECRET,
region: 'us-east-1',
partSize: 2000000,,
tmpDirPrefix: 'tmp',
namingFunction: fileNameFromUrl
});
const HEADERS = [
'Authorization',
'Content-Type',
'Location',
'Tus-Extension',
'Tus-Max-Size',
'Tus-Resumable',
'Tus-Version',
'Upload-Defer-Length',
'Upload-Length',
'Upload-Metadata',
'Upload-Offset',
'X-HTTP-Method-Override',
'X-Requested-With',
'Upload-Concat'
];
var corsOptions = {
allowedHeaders: HEADERS
}
const app = express();
const uploadApp = express();
uploadApp.all('*', cors(corsOptions), server.handle.bind(server));
app.use('/uploads', uploadApp);
const PORT = process.env.PORT || 9000;
spdy
.createServer(options, app)
.listen(PORT, (error) => {
if (error) {
console.error(error)
return process.exit(1)
} else {
console.log('Listening on port: ' + PORT + '.')
}
})
Using tus-js-client
:
uploader.js
import tus from 'tus-js-client'
const input = $("#uploadInput")
const $progress = $(".progress")
input.on("change", function(e) {
// Get the selected file from the input element
var file = e.target.files[0]
// Create a new tus upload
var upload = new tus.Upload(file, {
endpoint: "https://localhost:9000/uploads/",
retryDelays: [0, 1000, 3000, 5000],
chunkSize: 20000000,
removeFingerprintOnSuccess: true,
metadata: {
filename: file.name,
filetype: file.type
},
headers:{
"Upload-Concat": "partial"
},
onError: function(error) {
console.log("Failed because: " + error)
$progress.text("Failed")
},
onProgress: function(bytesUploaded, bytesTotal) {
var percentage = (bytesUploaded / bytesTotal * 100).toFixed(2)
console.log(percentage + "%")
$progress.text( percentage + "%" );
},
onSuccess: function() {
console.log("Download %s from %s", upload.file.name, upload.url)
}
})
// Start the upload
upload.start()
})
After all that my server console reports this error
Listening on port: 9000.
{ NotFound: null
at Request.extractError (/Users/xyz/paschaldev/mpa/upload-server/node_modules/aws-sdk/lib/services/s3.js:557:35)
at Request.callListeners (/Users/xyz/paschaldev/mpa/upload-server/node_modules/aws-sdk/lib/sequential_executor.js:105:20)
at Request.emit (/Users/xyz/paschaldev/mpa/upload-server/node_modules/aws-sdk/lib/sequential_executor.js:77:10)
at Request.emit (/Users/xyz/paschaldev/mpa/upload-server/node_modules/aws-sdk/lib/request.js:683:14)
at Request.transition (/Users/xyz/paschaldev/mpa/upload-server/node_modules/aws-sdk/lib/request.js:22:10)
at AcceptorStateMachine.runTo (/Users/xyz/paschaldev/mpa/upload-server/node_modules/aws-sdk/lib/state_machine.js:14:12)
at /Users/xyz/paschaldev/mpa/upload-server/node_modules/aws-sdk/lib/state_machine.js:26:10
at Request.<anonymous> (/Users/xyz/paschaldev/mpa/upload-server/node_modules/aws-sdk/lib/request.js:38:9)
at Request.<anonymous> (/Users/xyz/paschaldev/mpa/upload-server/node_modules/aws-sdk/lib/request.js:685:12)
at Request.callListeners (/Users/xyz/paschaldev/mpa/upload-server/node_modules/aws-sdk/lib/sequential_executor.js:115:18)
message: null,
code: 'NotFound',
region: null,
time: 2018-08-14T11:45:05.622Z,
requestId: '9B05E2BA054FA467',
extendedRequestId:
'hVUkIh8A6ldNOvJcccV2AF/+GJC5o+lm1tQb7zQP3sWXLTDVwogPXKhn2nDEfdFcAKVTNHkWvqY=',
cfId: undefined,
statusCode: 404,
retryable: false,
retryDelay: 59.78611958314186 }
[S3Store] getOffset: No file found.
[S3Store] Request "close" event was emitted,however S3 was expecting more data. Failing gracefully...
Issue Analytics
- State:
- Created 5 years ago
- Comments:8 (5 by maintainers)
Top Results From Across the Web
c# - Amazon S3 - Re-starting a failed upload - Stack Overflow
I am looking for advice on ways to try and restart a multi part upload of a file. That's a restart as in,...
Read more >If I restart my computer in the middle of an Amazon S3 upload ...
It gives the ability to “pause” and “resume” the upload without having to start over since any part that has completed would not...
Read more >Uploading and copying objects using multipart upload
After a successful complete request, the parts no longer exist. Your complete multipart upload request must include the upload ID and a list...
Read more >Networking Error on S3 management console - AWS re:Post
I am trying to upload new files to my S3 management console, but I get an ... I am unable to upload any...
Read more >How to resume Uncompleted Multipart Uploads - S3 Browser
Once initiated, the multipart upload never expire. You can either complete it by calling CompleteMultipartUpload when all parts are uploaded or abort it...
Read more >
Top Related Medium Post
No results found
Top Related StackOverflow Question
No results found
Troubleshoot Live Code
Lightrun enables developers to add logs, metrics and snapshots to live code - no restarts or redeploys required.
Start Free
Top Related Reddit Thread
No results found
Top Related Hackernoon Post
No results found
Top Related Tweet
No results found
Top Related Dev.to Post
No results found
Top Related Hashnode Post
No results found
i will double check as soon as possible.
Ok, thanks. I am sorry but I am not expert at tus-node-server’s S3Store. Maybe @rictorres has an idea?