From 704cb987c0d4063ad75af9e446fbe2557edd62b5 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Thu, 28 Sep 2023 14:38:45 +0330 Subject: [PATCH 1/2] Change name of fetchProjectsBySlugQuery field --- src/resolvers/projectResolver.test.ts | 44 ++++++++++---------- src/resolvers/qfRoundHistoryResolver.test.ts | 2 +- test/graphqlQueries.ts | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/resolvers/projectResolver.test.ts b/src/resolvers/projectResolver.test.ts index 882c4edad..7ae005553 100644 --- a/src/resolvers/projectResolver.test.ts +++ b/src/resolvers/projectResolver.test.ts @@ -29,7 +29,7 @@ import { fetchLikedProjectsQuery, fetchMultiFilterAllProjectsQuery, fetchNewProjectsPerDate, - fetchProjectsBySlugQuery, + fetchProjectBySlugQuery, fetchProjectUpdatesQuery, fetchSimilarProjectsBySlugQuery, getProjectsAcceptTokensQuery, @@ -5324,7 +5324,7 @@ function projectBySlugTestCases() { const result = await axios.post( graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, connectedWalletUserId: user!.id, @@ -5367,7 +5367,7 @@ function projectBySlugTestCases() { }).save(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, connectedWalletUserId: user!.id, @@ -5390,7 +5390,7 @@ function projectBySlugTestCases() { }); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, }, @@ -5448,7 +5448,7 @@ function projectBySlugTestCases() { await refreshProjectPowerView(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, }, @@ -5484,7 +5484,7 @@ function projectBySlugTestCases() { order: 1, }).save(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: projectWithCampaign.slug, }, @@ -5497,7 +5497,7 @@ function projectBySlugTestCases() { assert.isNotEmpty(project.campaigns); const projectWithoutCampaignResult = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: projectWithoutCampaign.slug, }, @@ -5527,7 +5527,7 @@ function projectBySlugTestCases() { order: 1, }).save(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: projectWithCampaign.slug, }, @@ -5541,7 +5541,7 @@ function projectBySlugTestCases() { assert.equal(project.campaigns[0].id, campaign.id); const projectWithoutCampaignResult = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { // and old project that I'm sure it would not be in the Newest campaign slug: SEED_DATA.FIRST_PROJECT.slug, @@ -5610,7 +5610,7 @@ function projectBySlugTestCases() { order: 1, }).save(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: projectWithCampaign.slug, }, @@ -5624,7 +5624,7 @@ function projectBySlugTestCases() { assert.equal(fetchedProject.campaigns[0].id, campaign.id); const projectWithoutCampaignResult = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: projectWithoutCampaign.slug, }, @@ -5659,7 +5659,7 @@ function projectBySlugTestCases() { order: 1, }).save(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: previousSlug, }, @@ -5755,7 +5755,7 @@ function projectBySlugTestCases() { await refreshProjectFuturePowerView(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, }, @@ -5850,7 +5850,7 @@ function projectBySlugTestCases() { await refreshProjectFuturePowerView(false); let result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, }, @@ -5865,7 +5865,7 @@ function projectBySlugTestCases() { await refreshProjectFuturePowerView(true); result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, }, @@ -5884,7 +5884,7 @@ function projectBySlugTestCases() { }); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: draftedProject.slug, }, @@ -5908,7 +5908,7 @@ function projectBySlugTestCases() { const result = await axios.post( graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: draftedProject.slug, connectedWalletUserId: SEED_DATA.FIRST_USER.id, @@ -5939,7 +5939,7 @@ function projectBySlugTestCases() { const result = await axios.post( graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: draftedProject.slug, }, @@ -5967,7 +5967,7 @@ function projectBySlugTestCases() { }); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project.slug, }, @@ -5991,7 +5991,7 @@ function projectBySlugTestCases() { const result = await axios.post( graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project.slug, connectedWalletUserId: SEED_DATA.FIRST_USER.id, @@ -6022,7 +6022,7 @@ function projectBySlugTestCases() { const result = await axios.post( graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: cancelledProject.slug, }, @@ -6080,7 +6080,7 @@ function projectBySlugTestCases() { await updateInstantBoosting(); const result = await axios.post(graphqlUrl, { - query: fetchProjectsBySlugQuery, + query: fetchProjectBySlugQuery, variables: { slug: project1.slug, }, diff --git a/src/resolvers/qfRoundHistoryResolver.test.ts b/src/resolvers/qfRoundHistoryResolver.test.ts index 3f4e9fa85..3fbf8b0a4 100644 --- a/src/resolvers/qfRoundHistoryResolver.test.ts +++ b/src/resolvers/qfRoundHistoryResolver.test.ts @@ -14,7 +14,7 @@ import moment from 'moment'; import { fillQfRoundHistory } from '../repositories/qfRoundHistoryRepository'; import axios from 'axios'; import { - fetchProjectsBySlugQuery, + fetchProjectBySlugQuery, getQfRoundHistoryQuery, } from '../../test/graphqlQueries'; diff --git a/test/graphqlQueries.ts b/test/graphqlQueries.ts index e041a1625..a6bf0808a 100644 --- a/test/graphqlQueries.ts +++ b/test/graphqlQueries.ts @@ -782,7 +782,7 @@ export const getQfRoundHistoryQuery = ` } `; -export const fetchProjectsBySlugQuery = ` +export const fetchProjectBySlugQuery = ` query ( $slug: String! ) { From 0325f7c7d674a8070af7d926630c5ee9792899a3 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Mon, 2 Oct 2023 17:54:26 +0330 Subject: [PATCH 2/2] Move caching project campaign slugs process to a separated worker related #1051 --- config/example.env | 3 ++ config/test.env | 4 ++ src/resolvers/projectResolver.test.ts | 3 ++ src/server/bootstrap.ts | 3 ++ src/services/campaignService.ts | 38 +++++++++---------- .../cronJobs/checkActiveStatusQfRounds.ts | 17 --------- .../updateProjectCampaignsCacheJob.ts | 37 ++++++++++++++++++ src/workers/cacheProjectCampaignsWorker.ts | 17 +++++++++ 8 files changed, 86 insertions(+), 36 deletions(-) create mode 100644 src/services/cronJobs/updateProjectCampaignsCacheJob.ts create mode 100644 src/workers/cacheProjectCampaignsWorker.ts diff --git a/config/example.env b/config/example.env index 9ecd2a5f9..9e61f4c03 100644 --- a/config/example.env +++ b/config/example.env @@ -205,3 +205,6 @@ QF_ROUND_ESTIMATED_MATCHING_CACHE_DURATION=60000 # OPTIONAL - default: Every 10 minutes PROJECT_CAMPAIGNS_CACHE_DURATION=600000 + +# OPTIONAL - default: */10 * * * * * ( Every 10 minutes) +PROJECT_CAMPAIGNS_CRONJOB_EXPRESSION=*/10 * * * * * diff --git a/config/test.env b/config/test.env index a5786e08c..71fce3f69 100644 --- a/config/test.env +++ b/config/test.env @@ -177,3 +177,7 @@ NUMBER_OF_BALANCE_AGGREGATOR_BATCH=7 QF_ROUND_ESTIMATED_MATCHING_CACHE_DURATION=1 # ! millisecond cache, if we increase cache in test ENV we might get some errors in tests PROJECT_CAMPAIGNS_CACHE_DURATION=1 + + +# OPTIONAL - default: */10 * * * * * ( Every 10 minutes) +PROJECT_CAMPAIGNS_CRONJOB_EXPRESSION=*/10 * * * * * diff --git a/src/resolvers/projectResolver.test.ts b/src/resolvers/projectResolver.test.ts index 7ae005553..84276636f 100644 --- a/src/resolvers/projectResolver.test.ts +++ b/src/resolvers/projectResolver.test.ts @@ -114,6 +114,7 @@ import { } from '../services/projectViewsService'; import { addOrUpdatePowerSnapshotBalances } from '../repositories/powerBalanceSnapshotRepository'; import { findPowerSnapshots } from '../repositories/powerSnapshotRepository'; +import { cacheProjectCampaigns } from '../services/campaignService'; const ARGUMENT_VALIDATION_ERROR_MESSAGE = new ArgumentValidationError([ { property: '' }, @@ -5526,6 +5527,7 @@ function projectBySlugTestCases() { photo: 'https://google.com', order: 1, }).save(); + await cacheProjectCampaigns(); const result = await axios.post(graphqlUrl, { query: fetchProjectBySlugQuery, variables: { @@ -5609,6 +5611,7 @@ function projectBySlugTestCases() { photo: 'https://google.com', order: 1, }).save(); + await cacheProjectCampaigns(); const result = await axios.post(graphqlUrl, { query: fetchProjectBySlugQuery, variables: { diff --git a/src/server/bootstrap.ts b/src/server/bootstrap.ts index f371922bc..51d498681 100644 --- a/src/server/bootstrap.ts +++ b/src/server/bootstrap.ts @@ -66,6 +66,8 @@ import { } from '../services/projectViewsService'; import { isTestEnv } from '../utils/utils'; import { runCheckActiveStatusOfQfRounds } from '../services/cronJobs/checkActiveStatusQfRounds'; +import { runUpdateProjectCampaignsCacheJob } from '../services/cronJobs/updateProjectCampaignsCacheJob'; +import { getAllProjectsRelatedToActiveCampaigns } from '../services/campaignService'; Resource.validate = validate; @@ -372,6 +374,7 @@ export async function bootstrap() { runInstantBoostingUpdateCronJob(); } await runCheckActiveStatusOfQfRounds(); + await runUpdateProjectCampaignsCacheJob(); } catch (err) { logger.error(err); } diff --git a/src/services/campaignService.ts b/src/services/campaignService.ts index ede5e7265..d3c31707d 100644 --- a/src/services/campaignService.ts +++ b/src/services/campaignService.ts @@ -9,6 +9,7 @@ import { ModuleThread, Pool } from 'threads'; import { ProjectResolverWorker } from '../workers/projectsResolverWorker'; import { QueryBuilder } from 'typeorm/query-builder/QueryBuilder'; import { findAllActiveCampaigns } from '../repositories/campaignRepository'; +import { logger } from '../utils/logger'; const projectFiltersCacheDuration = Number(process.env.PROJECT_FILTERS_THREADS_POOL_DURATION) || 60000; @@ -40,38 +41,37 @@ const createFetchCampaignProjectsQuery = ( return projectsQueryParams; }; -let projectCampaignCache: { [key: number]: string[] } | undefined; +let projectCampaignCache: { [key: number]: string[] } = {}; -export const getAllProjectsRelatedToActiveCampaigns = async (): Promise<{ +export const getAllProjectsRelatedToActiveCampaigns = (): { [key: number]: string[]; -}> => { +} => { // It returns all project and campaigns( excluding manuallySelectedCampaign) - if (projectCampaignCache) { - return projectCampaignCache; - } - projectCampaignCache = {}; + return projectCampaignCache; +}; + +export const cacheProjectCampaigns = async (): Promise => { + logger.debug('cacheProjectCampaigns() has been called'); + const newProjectCampaignCache = {}; const activeCampaigns = await findAllActiveCampaigns(); for (const campaign of activeCampaigns) { const projectsQueryParams = createFetchCampaignProjectsQuery(campaign); if (!projectsQueryParams) { - break; + continue; } const projectsQuery = filterProjectsQuery(projectsQueryParams); const projects = await projectsQuery.getMany(); for (const project of projects) { - projectCampaignCache[project.id] - ? projectCampaignCache[project.id].push(campaign.slug) - : (projectCampaignCache[project.id] = [campaign.slug]); + newProjectCampaignCache[project.id] + ? newProjectCampaignCache[project.id].push(campaign.slug) + : (newProjectCampaignCache[project.id] = [campaign.slug]); } } - const projectCampaignsCacheDuration = - Number(process.env.PROJECT_CAMPAIGNS_CACHE_DURATION) || 10 * 60 * 1000; - setTimeout(() => { - // We make it undefined every 10 minutes, to refresh it - projectCampaignCache = undefined; - }, projectCampaignsCacheDuration); - - return projectCampaignCache; + projectCampaignCache = newProjectCampaignCache; + logger.debug( + 'cacheProjectCampaigns() ended successfully, projectCampaignCache size ', + Object.keys(projectCampaignCache).length, + ); }; export const fillCampaignProjects = async (params: { diff --git a/src/services/cronJobs/checkActiveStatusQfRounds.ts b/src/services/cronJobs/checkActiveStatusQfRounds.ts index 584700b91..b9519bd1b 100644 --- a/src/services/cronJobs/checkActiveStatusQfRounds.ts +++ b/src/services/cronJobs/checkActiveStatusQfRounds.ts @@ -1,23 +1,6 @@ import config from '../../config'; import { logger } from '../../utils/logger'; import { schedule } from 'node-cron'; -import { - getPowerRound, - setPowerRound, -} from '../../repositories/powerRoundRepository'; -import { getRoundNumberByDate } from '../../utils/powerBoostingUtils'; -import { - refreshProjectPowerView, - refreshProjectFuturePowerView, - getBottomRank, -} from '../../repositories/projectPowerViewRepository'; -import { refreshUserProjectPowerView } from '../../repositories/userProjectPowerViewRepository'; -import { - copyProjectRanksToPreviousRoundRankTable, - deleteAllPreviousRoundRanks, - projectsThatTheirRanksHaveChanged, -} from '../../repositories/previousRoundRankRepository'; -import { getNotificationAdapter } from '../../adapters/adaptersFactory'; import { isTestEnv, sleep } from '../../utils/utils'; import { deactivateExpiredQfRounds, diff --git a/src/services/cronJobs/updateProjectCampaignsCacheJob.ts b/src/services/cronJobs/updateProjectCampaignsCacheJob.ts new file mode 100644 index 000000000..3797278ab --- /dev/null +++ b/src/services/cronJobs/updateProjectCampaignsCacheJob.ts @@ -0,0 +1,37 @@ +import config from '../../config'; +import { logger } from '../../utils/logger'; +import { schedule } from 'node-cron'; +import { isTestEnv } from '../../utils/utils'; +import { ModuleThread, Pool, spawn, Worker } from 'threads'; +import { CacheProjectCampaignsWorker } from '../../workers/cacheProjectCampaignsWorker'; + +// every 10 minutes +const cronJobTime = + (config.get('CACHE_PROJECT_CAMPAIGNS_CRONJOB_EXPRESSION') as string) || + '*/10 * * * *'; + +const projectsFiltersThreadPool: Pool< + ModuleThread +> = Pool( + () => spawn(new Worker('../../workers/cacheProjectCampaignsWorker')), // create the worker, +); +export const runUpdateProjectCampaignsCacheJob = () => { + // Run it first time to make sure it is cached + projectsFiltersThreadPool.queue(async worker => { + await worker.cacheSlugsOfCampaignProjects(); + }); + + logger.debug( + 'runUpdateProjectCampaignsCacheJob() has been called, cronJobTime', + cronJobTime, + ); + schedule(cronJobTime, async () => { + try { + projectsFiltersThreadPool.queue(async worker => { + await worker.cacheSlugsOfCampaignProjects(); + }); + } catch (e) { + logger.error('runUpdateProjectCampaignsCacheJob() error', e); + } + }); +}; diff --git a/src/workers/cacheProjectCampaignsWorker.ts b/src/workers/cacheProjectCampaignsWorker.ts new file mode 100644 index 000000000..87b3568a9 --- /dev/null +++ b/src/workers/cacheProjectCampaignsWorker.ts @@ -0,0 +1,17 @@ +// workers/auth.js +import { expose } from 'threads/worker'; +import { WorkerModule } from 'threads/dist/types/worker'; +import { cacheProjectCampaigns } from '../services/campaignService'; + +type ProjectsResolverWorkerFunctions = 'cacheSlugsOfCampaignProjects'; + +export type CacheProjectCampaignsWorker = + WorkerModule; + +const worker: CacheProjectCampaignsWorker = { + async cacheSlugsOfCampaignProjects() { + await cacheProjectCampaigns(); + }, +}; + +expose(worker);