Skip to content

Commit

Permalink
Restore parseNews to staging version
Browse files Browse the repository at this point in the history
  • Loading branch information
clari182 committed Dec 20, 2024
1 parent c2780d7 commit 451120b
Showing 1 changed file with 9 additions and 57 deletions.
66 changes: 9 additions & 57 deletions site/gatsby-site/netlify/functions/parseNews.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,60 +39,36 @@ exports.handler = async function (event) {
// Runs first with { cookies: false },
// then on error recurses with { cookies: true } as a fallback.
const getArticle = async (url, config) => {
const TIMEOUT_DURATION = 5000; // Timeout after 10 seconds

const timeoutPromise = new Promise((_, reject) =>
setTimeout(() => reject(new Error('Parser.parse timed out')), TIMEOUT_DURATION)
);

try {
const parserConfig = { contentType: 'markdown' };

if (config.cookies) {
parserConfig.html = await getHtmlWithCookies(url);
}

// Race the Parser.parse call with the timeout
const article = await Promise.race([
Parser.parse(url, parserConfig),
timeoutPromise
]);
const article = await Parser.parse(url, parserConfig);

return article;
} catch (error) {
console.log("Couldn't parse article", error.message);

if (error.message === 'Parser.parse timed out') {
console.log('The parser timed out. Retrying...');
}

if (config.cookies) {
console.log('Cookies were already enabled. Rethrowing error');
throw error; // Rethrow the error if cookies were already enabled
throw error;
} else {
console.log('Retrying with cookies enabled');
return await getArticle(url, { cookies: true });
}
}
};

const getHtmlWithCookies = async (url) => {
const axiosInstance = axios.create({
maxRedirects: 0,
withCredentials: true,
timeout: 5000, // Keep default timeout (10 seconds here, adjust if needed)
});
const axiosInstance = axios.create();

axiosInstance.defaults.maxRedirects = 0;
axiosInstance.defaults.withCredentials = true;
axiosInstance.defaults.credentials = 'same-origin';

console.log('Fetching URL with cookies:', url);
axiosInstance.interceptors.response.use(
(response) => response,
(error) => {
console.log('Intercepted error in interceptor:', error.message);

// Handle HTTP 3xx Redirects with Cookies
if (error.response && [301, 302].includes(error.response.status)) {
console.log('Handling redirect', error.response.status);

const redirectUrl = error.response.headers.location;

const Cookie = error.response.headers['set-cookie']
Expand All @@ -101,35 +77,11 @@ const getHtmlWithCookies = async (url) => {

return axiosInstance.get(redirectUrl, { headers: { Cookie } });
}

// Re-throw non-redirect errors
return Promise.reject(error);
}
);

try {
console.log('Fetching URL:', url);
const response = await axiosInstance.get(url);
console.log('Response received:', response.status);
return response.data;
} catch (error) {
// Log and handle Axios errors
console.error('Caught error:', error.message, error.code);

if (error.code === 'ECONNABORTED') {
console.error('Request timed out:', error.message);
} else if (error.response) {
console.error(
'Request failed with status:',
error.response.status,
error.response.data
);
} else {
console.error('Network or other error:', error.message);
}
const response = await axiosInstance.get(url);

// Re-throw the error for further handling
throw error;
}
return response.data;
};

0 comments on commit 451120b

Please sign in to comment.