Large File upload via REST API

[Mihai_Dobrescu1]

Hello,

I'm trying to upload a file to Marketo via REST API(nodejs app) and I've managed to do it for small size files with npm package "request-promise". 

const options = {

 method : 'POST',
 uri: uri,
 formData:{
 name:filename,
 folder:folder,
 file: {
 value: fs.createReadStream(path, {highWaterMark: 256 * 1024}),
 options: {
 filename: filename,
 contentType: mimeType
 }
 }
 }
};‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍

The issue comes when I'm trying to upload a larger file and I get "413 Request Entity Too Large" errors. It looks like the stream of file is too big for Marketo to handle. The file has 11mb so it's not over Marketo's limit.

So now I'm trying to use "https" npm module so I can write the request myself but I'm getting a 611 Marketo Error(unhandled error)

 The code for the request looks like this:

let mimeType = mime.lookup(filename);

var file = {
 value: fs.createReadStream(pathFile + "/data/process/scanned/" + filename),
 options: {
 filename: filename,
 contentType: mimeType
 }
};

var options = {
 "method": "POST",
 "hostname": clientConfig.host2,
 "path": "/rest/asset/v1/files.json?access_token=" + result.access_token,
 "headers": {
 "content-type": "multipart/form-data; boundary=----WebKitFormBoundary8MA4YWxkTrZu0gW"
 }
};


var strFile = '------WebKitFormBoundary8MA4YWxkTrZu0gW\r\nContent-Disposition: form-data; name = "file"; filename = "' + filename + '"\r\n' + 'Content-Type:' + mimeType + '\r\n' + JSON.stringify(file);
var strFolder = '------WebKitFormBoundary8MA4YWxkTrZu0gW\r\nContent-Disposition: form-data; name = "folder" \r\n' + JSON.stringify(folder);
str2 = strFile + "\r\n" + strFolder + "\r\n" + "------WebKitFormBoundary8MA4YWxkTrZu0gW-"‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍;‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍

 req.write(str2);
 req.end();
‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍

On runtime i get this request:

------WebKitFormBoundary8MA4YWxkTrZu0gW
Content-Disposition: form-data; name = "file"; filename = "fi.zip"
Content-Type:application/zip
{"value":{"_readableState":{"objectMode":false,"highWaterMark":65536,"buffer":{"head":null,"tail":null,"length":0},"length":0,"pipes":null,"pipesCount":0,"flowing":null,"ended":false,"endEmitted":false,"reading":false,"sync":true,"needReadable":false,"emittedReadable":false,"readableListening":false,"resumeScheduled":false,"emitClose":false,"destroyed":false,"defaultEncoding":"utf8","awaitDrain":0,"readingMore":false,"decoder":null,"encoding":null},"readable":true,"_events":{},"_eventsCount":1,"path":"C:\\LEADFABRIC\\kanthal\\fileUpload/data/process/scanned/fi.zip","fd":null,"flags":"r","mode":438,"end":null,"autoClose":true,"bytesRead":0,"closed":false},"options":{"filename":"fi.zip","contentType":"application/zip"}}
------WebKitFormBoundary8MA4YWxkTrZu0gW
Content-Disposition: form-data; name = "folder" 
{"id":"1400","type":"folder"}
------WebKitFormBoundary8MA4YWxkTrZu0gW-‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍

I've tried to make it so it's similar to Marketo documentation on file upload(https://developers.marketo.com/rest-api/assets/files/#create_and_update ) but can't seem to make it work.

Any ideas how to solve this issue?

Thank you

[SanfordWhiteman]

At a glance, your first snippet is fine. The limits via REST simply aren't the same as the limits via UI.

[Mihai_Dobrescu1]

Hi Sanford,

I know, but I've managed to upload that file on other APIs and there were no issues, so probably the issue is on writing the streams.

So for the moment I gave up on writing those streams and use a formdata object

         var formData = new FormData();
                formData.append("file", JSON.stringify(fs.createReadStream(pathFile + "/data/process/scanned/" + filename)),file.options);
                formData.append("name", filename);
                formData.append("folder", JSON.stringify(folder));‍‍‍‍

req.write(JSON.stringify(formData));‍‍‍‍‍‍‍‍‍‍‍‍

The stream looks like this:

  _streams:
   [ '----------------------------156825794513753139646442\r\nContent-Disposition: form-data; name="file"; filename="fi.zip"\r\nContent-Type: application/zip\r\n\r\n',
     '{"_readableState":{"objectMode":false,"highWaterMark":65536,"buffer":{"head":null,"tail":null,"length":0},"length":0,"pipes":null,"pipesCount":0,"flowing":null,"ended":false,"endEmitted":false,"reading":false,"sync":true,"needReadable":false,"emittedReadable":false,"readableListening":false,"resumeScheduled":false,"emitClose":false,"destroyed":false,"defaultEncoding":"utf8","awaitDrain":0,"readingMore":false,"decoder":null,"encoding":null},"readable":true,"_events":{},"_eventsCount":1,"path":"C:\\\\LEADFABRIC\\\\kanthal\\\\fileUpload/data/process/scanned/fi.zip","fd":null,"flags":"r","mode":438,"end":null,"autoClose":true,"bytesRead":0,"closed":false}',
     [Function: bound ],
     '----------------------------156825794513753139646442\r\nContent-Disposition: form-data; name="name"\r\n\r\n',
     'fi.zip',
     [Function: bound ],
     '----------------------------156825794513753139646442\r\nContent-Disposition: form-data; name="folder"\r\n\r\n',
     '{"id":"1400","type":"folder"}',
     [Function: bound ] ],
  _currentStream: null,
  _boundary: '--------------------------156825794513753139646442' }‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍

But it's strange that the response I get from Marketo is that "file is mandatory". So it looks like the file is not visible in the stream which is weird.

Even if I stringify or not the second parameter on the "file" append, the answer is the same: ""file is mandatory". 

Any ideas why this happens?

Thank you

Later edit:

I've managed to fix the issue on the request but now I'm still getting "413 Request Entity Too Large"

I've added this line in the request option object:

headers: formData.getHeaders()‍

and used the pipe option on the form

//req.write(JSON.stringify(formData));
formData.pipe(req);‍‍

From what I've checked on 413 error is for some, the issue is solved by increasing the limit on the body.parser. But in my case it didn't.

Could it be an issue with Marketo?

[Jep_Castelein2]

The maximum file size for file uploads via the REST API is 10 Mb. So unfortunately you will have to upload those files manually via the UI. Files up to 10 Mb should upload with no issue. 

[Mihai_Dobrescu1]

Thank you for the information. Unfortunately the uploads are made through a custom application, so the upload via UI is not an option.

[Jep_Castelein2]

Unfortunately there is nothing that can be done about that limitation: the 10Mb limit is hard. If you are going the custom route, maybe upload files > 10Mb to a public S3 bucket (or other file hosting service) and save the URL in a Marketo Program Token or "My Token" on a Marketing Activities Folder?