Skip to content

Commit

Permalink
fix: finish lisftwing api post call
Browse files Browse the repository at this point in the history
  • Loading branch information
@justiceotuya
justiceotuya committed Oct 16, 2023
1 parent 1e5dcd6 commit 6b039f3
Show file tree
Hide file tree
Showing 2 changed files with 87 additions and 77 deletions.
152 changes: 76 additions & 76 deletions app/assets/javascripts/actions/article_finder_action.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { queryUrl, categoryQueryGenerator, pageviewQueryGenerator, pageAssessmen
import { ORESSupportedWiki, PageAssessmentSupportedWiki } from '../utils/article_finder_language_mappings.js';
import { fetchWikidataLabels } from './wikidata_actions';
import request from '../utils/request';
import logErrorMessage from '../utils/log_error_message';

const mediawikiApiBase = (language, project) => {
if (project === 'wikidata') {
Expand All @@ -20,13 +21,6 @@ const liftWingApiBase = (language, project) => {
return `${LIFT_WING_SERVER_URL}/service/lw/inference/v1/models/${project_code}-${project_quality_model}:predict`;
};

const oresApiBase = (language, project) => {
if (project === 'wikidata') {
return `https://ores.wikimedia.org/v3/scores/${project}wiki`;
}
return `https://ores.wikimedia.org/v3/scores/${language}wiki`;
};

const limit = promiseLimit(10);

export const updateFields = (key, value) => (dispatch) => {
Expand All @@ -46,72 +40,75 @@ export const sortArticleFinder = (key) => {
};
};

export const fetchCategoryResults = (category, wiki, cmcontinue = '', continueResults = false) => (dispatch, getState) => {
if (!continueResults) {
dispatch({
type: INITIATE_SEARCH,
});
} else {
dispatch({
type: UPDATE_FINDER_FIELD,
data: {
key: 'fetchState',
value: 'ARTICLES_LOADING',
}
});
}
return getDataForCategory(`Category:${category}`, wiki, cmcontinue, 0, dispatch, getState);
};
export const fetchCategoryResults = (category, wiki, cmcontinue = '', continueResults = false) =>
(dispatch, getState) => {
if (!continueResults) {
dispatch({
type: INITIATE_SEARCH,
});
} else {
dispatch({
type: UPDATE_FINDER_FIELD,
data: {
key: 'fetchState',
value: 'ARTICLES_LOADING',
},
});
}
return getDataForCategory(
`Category:${category}`,
wiki,
cmcontinue,
0,
dispatch,
getState
);
};

const getDataForCategory = (category, wiki, cmcontinue, namespace = 0, dispatch, getState) => {
const getDataForCategory = (
category,
wiki,
cmcontinue,
namespace = 0,
dispatch,
getState
) => {
const query = categoryQueryGenerator(category, cmcontinue, namespace);
return limit(() => queryUrl(mediawikiApiBase(wiki.language, wiki.project), query))
.then((data) => {
// if (depth > 0) {
// depth -= 1;
// getDataForSubCategories(category, depth, namespace, dispatch, getState);
// }
dispatch({
type: RECEIVE_CATEGORY_RESULTS,
data: data,
});
return data.query.categorymembers;
})
.then((data) => {
fetchPageAssessment(data, wiki, dispatch, getState);
})
.catch(response => (dispatch({ type: API_FAIL, data: response })));
return limit(() =>
queryUrl(mediawikiApiBase(wiki.language, wiki.project), query)
)
.then((data) => {
// if (depth > 0) {
// depth -= 1;
// getDataForSubCategories(category, depth, namespace, dispatch, getState);
// }
dispatch({
type: RECEIVE_CATEGORY_RESULTS,
data: data,
});
return data.query.categorymembers;
})
.then((data) => {
fetchPageAssessment(data, wiki, dispatch, getState);
})
.catch(response => dispatch({ type: API_FAIL, data: response }));
};

// export const findSubcategories = (category) => {
// const subcatQuery = categoryQueryGenerator(category, 14);
// return limit(() => queryUrl(mediawikiApiBase, subcatQuery))
// .then((data) => {
// return data.query.categorymembers;
// });
// };

// const getDataForSubCategories = (category, depth, namespace, dispatch, getState) => {
// return findSubcategories(category)
// .then((subcats) => {
// subcats.forEach((subcat) => {
// getDataForCategory(subcat.title, depth, namespace, dispatch, getState);
// });
// });
// };

const fetchPageViews = (articlesList, wiki, dispatch, getState) => {
const promises = chunk(articlesList, 5).map((articles) => {
const promises = chunk(articlesList, 10).map((articles) => {
const query = pageviewQueryGenerator(map(articles, 'pageid'));
return limit(() => queryUrl(mediawikiApiBase(wiki.language, wiki.project), query))
.then(data => data.query.pages)
.then((data) => {
dispatch({
type: RECEIVE_ARTICLE_PAGEVIEWS,
data: data
});
})
.catch(response => (dispatch({ type: API_FAIL, data: response })));
return limit(() =>
queryUrl(mediawikiApiBase(wiki.language, wiki.project), query)
)
.then(data => data.query.pages)
.then((data) => {
dispatch({
type: RECEIVE_ARTICLE_PAGEVIEWS,
data: data,
});
})
.catch(response => dispatch({ type: API_FAIL, data: response }));
});
const sort = getState().articleFinder.sort;
let desc = false;
Expand All @@ -120,8 +117,7 @@ const fetchPageViews = (articlesList, wiki, dispatch, getState) => {
} else if (!sort.sortKey) {
desc = true;
}
Promise.all(promises)
.then(() => {
Promise.all(promises).then(() => {
dispatch({
type: SORT_ARTICLE_FINDER,
key: sort.key,
Expand All @@ -132,8 +128,11 @@ const fetchPageViews = (articlesList, wiki, dispatch, getState) => {
};

const fetchPageAssessment = (articlesList, wiki, dispatch, getState) => {
if (PageAssessmentSupportedWiki[wiki.project] && includes(PageAssessmentSupportedWiki[wiki.project], wiki.language)) {
const promises = chunk(articlesList, 1).map((articles) => {
if (
PageAssessmentSupportedWiki[wiki.project]
&& includes(PageAssessmentSupportedWiki[wiki.project], wiki.language)
) {
const promises = chunk(articlesList, 10).map((articles) => {
const query = pageAssessmentQueryGenerator(map(articles, 'title'));

return limit(() =>
Expand All @@ -149,8 +148,7 @@ const fetchPageAssessment = (articlesList, wiki, dispatch, getState) => {
.catch(response => dispatch({ type: API_FAIL, data: response }));
});

Promise.all(promises)
.then(() => {
Promise.all(promises).then(() => {
fetchPageRevision(articlesList, wiki, dispatch, getState);
});
} else {
Expand All @@ -159,7 +157,10 @@ const fetchPageAssessment = (articlesList, wiki, dispatch, getState) => {
};

const fetchPageRevision = (articlesList, wiki, dispatch, getState) => {
if (includes(ORESSupportedWiki.languages, wiki.language) && includes(ORESSupportedWiki.projects, wiki.project)) {
if (
includes(ORESSupportedWiki.languages, wiki.language)
&& includes(ORESSupportedWiki.projects, wiki.project)
) {
const promises = chunk(articlesList, 1).map((articles) => {
const query = pageRevisionQueryGenerator(map(articles, 'title'));
return limit(() =>
Expand All @@ -178,8 +179,7 @@ const fetchPageRevision = (articlesList, wiki, dispatch, getState) => {
})
.catch(response => dispatch({ type: API_FAIL, data: response }));
});
Promise.all(promises)
.then(() => {
Promise.all(promises).then(() => {
fetchPageViews(articlesList, wiki, dispatch, getState);
});
} else {
Expand All @@ -201,7 +201,7 @@ const fetchPageRevisionScore = async (revids, wiki, dispatch) => {


try {
const response = await promiseLimit(2)(() =>
const response = await promiseLimit(5)(() =>
request(API_URL, {
method: 'POST',
headers: {
Expand Down
12 changes: 11 additions & 1 deletion app/assets/javascripts/reducers/article_finder.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,17 @@ export default function articleFinder(state = initialState, action) {
const newStateArticles = cloneDeep(state.articles);
forEach(action.data.data, (scores, revid) => {
const revScore = reduce(ORESWeights[`${action.data.project === 'wikidata' ? 'wikidata' : action.data.language}`], (result, value, key) => {
return result + (value * scores[`${action.data.project === 'wikidata' ? 'itemquality' : 'wp10'}`].score.probability[key]);
return (
result
+ value
* scores[
`${
action.data.project === 'wikidata'
? 'itemquality'
: 'articlequality'
}`
].score.probability[key]
);
}, 0);
const article = find(newStateArticles, { revid: parseInt(revid) });
article.revScore = Math.round(revScore * 100) / 100;
Expand Down

0 comments on commit 6b039f3

Please sign in to comment.