For anyone interested, I was able to capture my data at the end of the redirected url with authentication in a workflow function like this
const fetch = require("node-fetch");
const apiKey = retoolContext.configVars.myKey;
const apiSecret = retoolContext.configVars.mySecret;
const authKey = Basic ${btoa(${apiKey}:${apiSecret})};
const url = myUrl;
const response = await fetch(url, {
method: 'GET',
headers: {
'Authorization': authKey,
'Content-Type': 'application/json'
},
redirect: 'manual'
});
//just capturing the redirect url here
const locationUrl = response.headers.get('location');
if(!locationUrl) return null;
const fileResponse = await fetch(locationUrl, {
method: 'GET',
headers: {
'Authorization': authKey,
'Content-Type': 'application/json'
}
});
//in my case, the file came in as binary so converting to base64 for transporting to other workflow blocks
const buffer = await fileResponse.arrayBuffer();
return {
base64Data: Buffer.from(buffer).toString('base64'),
fileName: fileName,
type: fileType
};