Retry multiple fetch - fetch

How do I retry this fetch x times if it fails?
The code is based on this article: https://dmitripavlutin.com/javascript-fetch-async-await/
async function fetchData() {
const [firstResponse, secondResponse] = await Promise.all([
fetch(firstUrl),
fetch(secondUrl),
]);
const first = await firstResponse.json();
const second = await secondResponse.json();
return [first, second];
}
fetchData()
.then(([first, second]) => {
console.log("success");
})
.catch((error) => {
console.log("error");
});

Since the requests are independent of each other, I'd have a utility function that will retry X times and then use that in the Promise.all. I'd also have a utility function for fetching JSON that handles the fetch API footgun where it doesn't check HTTP success (see my blog post here). So something along these lines:
// Fetch JSON
function fetchJSON(...args) {
const response = await fetch(...args);
if (!response.ok) {
throw new Error(`HTTP error ${response.status}`);
}
return response.json();
}
// Fetch JSON with up to `retries` retries
async fetchJSONWithRetry(retries, ...args) {
while (retries > 0) {
try {
const result = await fetchJSON(...args);
return result;
} catch (e) {
if (--retries === 0) {
throw e;
}
}
}
}
// Your `fetchData`
async function fetchData(retries = 5) {
const [first, second] = await Promise.all([
fetchJSONWithRetry(retries, firstUrl),
fetchJSONWithRetry(retries, secondUrl),
]);
return [first, second];
}

Related

Can i dispatch many actions in getServersideprops?

In my social media app in Home page i want to dispatch 3 actions from my api:
posts , users , userDetails
But this may show an error(500) on vercel because the request takes a lot of time to get all these data.
vercel Error
this error will not appear again after refreshing the page !!!
i think that's because the request takes a lot of time to get all the data.
-> getServersideProps Code
export const getServerSideProps = wrapper.getServerSideProps(
store => async (context) =>
{
const {req} = context
const session = await getSession({ req });
await store.dispatch(fetchPostsAction());
await store.dispatch(fetchUsersAction(4));
await store.dispatch(LoggedInUserAction({email:session.user.email}));
})
-> fetchPostsAction Code
"post/list",
async (_, { rejectWithValue, getState, dispatch }) => {
try
{
let link = `${URL}/api/posts`;
const { data } = await axios.get(link,{
headers: { "Accept-Encoding": "gzip,deflate,compress" }
});
console.log("#2 got the data",data)
return data;
} catch (error) {
if (!error?.response) throw error;
return rejectWithValue(error?.response?.data);
}
}
);
-> extraReducer Code
builder.addCase(createpostAction.pending, (state, action) => {
state.createPostLoading = true;
});
builder.addCase(createpostAction.fulfilled, (state, action) => {
state.postLists = [...state.postLists, action.payload.post].sort((a, b) => b.createdAt > a.createdAt ? 1 : -1)
state.createPostLoading = false;
state.isCreated = true;
state.appErr = null;
state.serverErr = null;
});
builder.addCase(createpostAction.rejected, (state, action) => {
state.createPostLoading = false;
state.appErr =
action?.payload?.message || action?.payload?.error?.message;
state.serverErr = action?.error?.message;
});
-> get posts from api Code
handler.get(async (req, res) =>
{
await db.connect();
try {
const posts = await Post.find().populate({
path: 'user',
model: 'User',
}).populate({
path:'comments',
options: {sort: {'createdAt' : -1} }
}).sort('-createdAt')
res.status(200).json({
success:true,
posts
});
} catch (err) {
res.status(500).json(err.message)
}
await db.disconnect();
})
so what is the best way to fetch all these data in next js ?
I hope there is a way to solve this problem

waitForResponse does not work in the cluster.tasks callback

I need to open 20 pages parallelly and click on a button then wait for a response after that get the data from a tag. and my code is:
async function getPageData(links) {
return new Promise(async (resolve, reject) => {
try {
const cluster = await Cluster.launch({
concurrency: Cluster.CONCURRENCY_PAGE,
maxConcurrency: 200,
monitor: true,
});
let allData = [];
await cluster.task(async function getData({ page, data: url }) {
await page.goto(url, {
waitUntil: 'networkidle2',
});
const buttonQuery = 'button[role=tab]:first-child';
const buttonElement = await page.waitForSelector(buttonQuery);
await buttonElement.click(buttonElement);
await page.waitForResponse('https://XXX'); // the problem is here
const data = await page.evaluate(getList);
const [oscillators, summary, movingAverage] = data;
allData.push({ oscillators, summary, movingAverage });
});
links.map(async function addQueue(link) {
cluster.queue(link);
});
await cluster.idle();
await cluster.close();
resolve(allData);
} catch (e) {
reject(e);
}
});
but it just work for the first time and ignore the rest of the tasks. but when I remove page.waitForResponse() the all tasks will be run as expected.
How can I make all tasks wait for their response then extract the data?

Jasmine 4: Async function did not complete within 5000ms issue

I have an existing async function:
async doJSONGetRequest(getUrl, accessToken) {
return new Promise(function(resolve, reject) {
const reqHeaders = {
'Content-Type': 'application/json',
'Authorization': `Bearer ${accessToken}`,
};
console.info('url = ' + getUrl);
request.get({
url: getUrl,
headers: reqHeaders,
}, function(err, response) {
if (err) return reject(err);
try {
// console.debug(`response = ${response.body}`);
const parsed = JSON.parse(response.body);
return resolve(parsed);
} catch (err) {
return reject(err);
}
});
});
}
}
I'm trying to test it with Jasmine(v4).
Of course, I don't want this thing to actually make an HTTP request, so I tried rigging up a spy on the 'request' package's 'get' function in the 'beforeAll' section:
describe('RAPIDAPIService', function() {
beforeAll(async function() {
spyOn(request, 'get')
.and
.callFake(async (parameters) => {
if (parameters.url === 'http://localhost/api/getSomething') {
const rsp = {};
rsp.body = 'good stuff';
return rsp;
} else if (parameters.url === 'http://localhost/api/whoops') {
return new Error('401 not found');
} else {
return null;
}
});
});
it('doJSONGetRequest should run successfully', async () => {
expect(api.doJSONGetRequest).toBeDefined();
const res = await api.doJSONGetRequest('http://localhost/api/getSomething', '12345678');
expect(data).toEqual('good stuff');
});
it('doJSONGetRequest should resolve errors properly', async () => {
expect(api.doJSONGetRequest).toBeDefined();
const res = await api.doJSONGetRequest('http://localhost/api/whoops', '12345678');
const expectedError = new Error('401 not found');
expect(res).toEqual(expectedError);
});
Console log statements seem to indicate that I'm actually getting past / returning something from my "await" calls in the "it" tests. But the spies are actually working / detecting that the url's have been called.
(Note that I'm not including here other tests in the same file that do not make asynchronous calls and ARE working... just so you know that there's no problem accessing the actual "api" library and its functions.)
These two tests keep failing with "Error: Timeout - Async function did not complete within 5000ms". And like I said, it seems like they're not returning back to the tests from their calls to the doJSONGetRequest function.
Any thoughts?
Thanks!
I am thinking the issue is the mocking. request.get seems to take two parameters and I am thinking you need to call the 2nd parameter (callback function) once you are done so the resolve can be called.
Try this:
spyOn(request, 'get')
.and
// add callbackFunction as 2nd argument
.callFake((parameters, callbackFunction) => {
if (parameters.url === 'http://localhost/api/getSomething') {
const rsp = {};
rsp.body = 'good stuff';
callbackFunction(null, rsp);
} else if (parameters.url === 'http://localhost/api/whoops') {
callbackFunction({ error: '401 not found' }, {});
} else {
callbackFunction(null, null);
}
});

Returning a value from an asynchronous callback (Node.JS)

So, I have my route which console.logs 'undefined':
router.get("/validate-pin", async (req, res) => {
// restrict when done
try {
const { userId, pin } = req.query;
const isActivePin = await pinsDB.compareActivePin(userId, pin);
console.log(isActivePin)
return res.status(200).json(isActivePin);
} catch (error) {
console.log(error);
res.status(500).json({ error: "db error: ", error });
}
});
I have my compareActivePin method, which logs out the 'res' parameter, but for some reason doesn't return it:
async function compareActivePin(userId, received_pin) {
const active_pin = await db("account_pins").where({ userId, isActive: true });
const pinIsValidated = bcrypt.compareSync(
received_pin,
active_pin[0].account_pin
);
if (pinIsValidated) {
let skLocation = await db("sks").where({ userId }).select("url");
await readKey(skLocation[0].url, (res) => {
// console.log(res);
return res;
});
} else return false;
}
And I have my readKey method, which actually grabs the data I want my compareActivePin to return. This works like a charm.
const readKey = async (key, callback) => {
const aws = require("aws-sdk");
aws.config.update({
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
region: "us-east-2",
});
const s3 = new aws.S3();
const getParams = { Bucket: process.env.SK_BUCKET, Key: `${key}.txt` };
await s3.getObject(getParams, (err, data) => {
if (err) return err;
return callback(data.Body.toString());
});
};
So, just to recap. When I hit my endpoint, I pass in a userId and pin (strings). This calls the compareActivePin method which validates the pin and then, if the pin is valid, it then calls readKey, which grabs the file from S3 and returns the text within the file.
Like I said, I'm able to log it out to the console from within the readKey callback, but when I try to log it out as the returned value from the route, it comes back undefined.
Hoping someone could point me in the right direction.
Thanks...
I ended up answering my own question. I don't think it's possible to get a return value from the callback, so I ended up paring down the call from the database and sending the response from the readKey function using the router response object, like so:
//CompareActivePin Function
async function compareActivePin(userId, received_pin) {
const active_pin = await db("account_pins").where({ userId, isActive: true });
const pinIsValidated = bcrypt.compareSync(
received_pin,
active_pin[0].account_pin
);
return pinIsValidated;
}
//Router Call
router.get("/validate-pin", async (req, res) => {
// restrict when done
try {
const { userId, pin } = req.query;
const isActivePin = await pinsDB.compareActivePin(userId, pin);
if (isActivePin) {
let skLocation = await skDB.findUrl(userId);
readKeyFunc(skLocation[0].url, (result) => {
return res.status(200).json({ confirmed: isActivePin, key: result });
});
} else return res.status(401).json({ confirmed: isActivePin, key: null });
} catch (error) {
res.status(500).json({ error: "db error: ", error });
}
});
This also goes a long way toward keeping my database methods pure and separating my concerns.
Thanks, StackOverflow!

Redux Thunk Common Actions - Standalone + Combined

Implementing the pattern Dan (#gaearon) demonstrated here, I was just wondering the best way to have the common action work standalone in the following scenario:
const commonAction = () => {
return async (dispatch, getState, api) => {
try {
const response = await api.get('/resource');
dispatch(success('SUCCESS', response.data));
} catch(error) {
dispatch(error('ERROR', error));
throw error; //this is the problem
}
}
const combinedAction = () => {
return async (dispatch, getState, api) => {
try {
await dispatch(commonAction());
const otherResponse = await api.get('/otherResource');
dispatch(success('COMBINED_SUCCESS', otherResponse.data));
} catch(error) {
dispatch(error('COMBINED_ERROR', error));
}
}
The above works in that if either fails, the COMBINED_ERROR will be dispatched, due to the commonAction re-throwing it's error. However, if I wanted to use commonAction by itself, I would get an Unhandled promise rejection error due to the re-throw.
One thought I had was an optional bool isStandAlone = true to determine whether to re-throw or not, i.e.
const commonAction = (isStandAlone = true) => {
return async (dispatch, getState, api) => {
try {
const response = await api.get('/resource');
dispatch(success('SUCCESS', response.data));
} catch(error) {
dispatch(error('ERROR', error));
if(!isStandAlone) {
throw error;
}
}
}
And then in the combinedAction I would just:
await dispatch(commonAction(false));
Is this a code smell? Is there a better and/or different way to approach this?

Resources