From 11b6e016f88a51cb6c03f765f64cfccf59a26895 Mon Sep 17 00:00:00 2001 From: Jesiel Viana Date: Thu, 7 Dec 2023 17:14:25 -0300 Subject: [PATCH] =?UTF-8?q?add=20loader=20nas=20p=C3=A1ginas=20de=20busca?= =?UTF-8?q?=20e=20implementa=C3=A7=C3=A3o=20da=20exporta=C3=A7=C3=A3o=20pa?= =?UTF-8?q?ra=20cvs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 3 +- package.json | 10 +- public/locales/en/common.json | 4 +- public/locales/pt-BR/common.json | 4 +- src/components/AllIndexVisNetwork.tsx | 1 - src/components/BasicSearchBox.tsx | 1 - src/components/DownloadModal.tsx | 79 ++++++++++++++ .../indicators/GroupsIndicators.tsx | 10 +- .../indicators/JornalsIndicators.tsx | 4 +- .../indicators/OrgUnitIndicators.tsx | 6 +- .../indicators/PatentsIndicators.tsx | 10 +- .../indicators/PeopleIndicators.tsx | 7 +- .../indicators/ProgramsIndicators.tsx | 4 +- .../indicators/PublicationsIndicators.tsx | 6 +- .../indicators/SoftwaresIndicators.tsx | 6 +- src/components/indicators/query/Query.tsx | 49 +++++---- src/pages/api/bulk-download.ts | 103 ++++++++++++++++++ src/pages/api/download.ts | 14 +++ src/pages/api/search.ts | 1 - src/pages/groups.tsx | 12 +- src/pages/institutions.tsx | 13 ++- src/pages/journals.tsx | 13 ++- src/pages/patents.tsx | 13 ++- src/pages/people.tsx | 22 +++- src/pages/programs.tsx | 13 ++- src/pages/publications.tsx | 18 ++- src/pages/software.tsx | 13 ++- src/services/BulkDownloadService.ts | 18 +++ src/styles/Loader.module.css | 1 + src/styles/globals.scss | 15 ++- src/types/Entities.ts | 1 + yarn.lock | 94 +++++++++++++++- 32 files changed, 482 insertions(+), 86 deletions(-) create mode 100644 src/components/DownloadModal.tsx create mode 100644 src/pages/api/bulk-download.ts create mode 100644 src/pages/api/download.ts create mode 100644 src/services/BulkDownloadService.ts diff --git a/.env.example b/.env.example index 3cb1e26..3bd8cc8 100644 --- a/.env.example +++ b/.env.example @@ -19,4 +19,5 @@ INDEX_JOURNAL="release2-journal" INDEX_PROGRAM="release2-program" INDEX_PATENT="release2-patent" INDEX_GROUP="release2-group" -INDEX_SOFTWARE="release2-software" \ No newline at end of file +INDEX_SOFTWARE="release2-software" +DOWNLOAD_FOLDER_PATH="/tmp/brcris" \ No newline at end of file diff --git a/package.json b/package.json index 5ee8c35..702f261 100755 --- a/package.json +++ b/package.json @@ -11,15 +11,13 @@ "deploy": "yarn && yarn build && pm2 start yarn --name 'brcris-nextjs' --interpreter bash -- start" }, "dependencies": { - "@elastic/react-search-ui": "^1.18.1", - "@elastic/search-ui-elasticsearch-connector": "^1.18.1", - "@types/react-google-recaptcha": "^2.1.5", - "@types/react-tagcloud": "^1.1.7", "bootstrap": "^5.2.2", "chart.js": "^3.9.1", "es7": "npm:@elastic/elasticsearch@7", "i18next": "^22.4.14", "js-cookie": "^3.0.5", + "json-2-csv": "^5.0.1", + "jszip": "^3.10.1", "next": "12.3.1", "next-i18next": "^13.2.2", "node-fetch": "^3.3.1", @@ -39,12 +37,16 @@ "swr": "^1.3.0" }, "devDependencies": { + "@elastic/react-search-ui": "^1.18.1", + "@elastic/search-ui-elasticsearch-connector": "^1.18.1", "@types/js-cookie": "^3.0.3", "@types/node": "18.7.23", "@types/nodemailer": "^6.4.7", "@types/react": "18", "@types/react-csv": "^1.1.3", "@types/react-dom": "18", + "@types/react-google-recaptcha": "^2.1.5", + "@types/react-tagcloud": "^1.1.7", "@types/vis": "^4.21.24", "@typescript-eslint/eslint-plugin": "^5.38.1", "eslint": "8.24.0", diff --git a/public/locales/en/common.json b/public/locales/en/common.json index e48dadc..094f1ff 100644 --- a/public/locales/en/common.json +++ b/public/locales/en/common.json @@ -159,5 +159,7 @@ "For example, if you want to search for publications on Computing in 2018, excluding master thesis: go to the ": "For example, if you want to search for publications on 'Computing' in 2018, excluding master thesis: go to the ", "index page and use the query below": "index page and use the query below", "See below for the full list of search fields that can be used for the index of": "See below for the full list of search fields that can be used for the index of", - "(all:Computing) AND (publicationDate:2018) AND NOT (type:master thesis)": "(all:Computing) AND (publicationDate:2018) AND NOT (type:master thesis)" + "(all:Computing) AND (publicationDate:2018) AND NOT (type:master thesis)": "(all:Computing) AND (publicationDate:2018) AND NOT (type:master thesis)", + "Download": "Download", + "Export csv": "Export csv" } diff --git a/public/locales/pt-BR/common.json b/public/locales/pt-BR/common.json index d2de1bc..1612926 100644 --- a/public/locales/pt-BR/common.json +++ b/public/locales/pt-BR/common.json @@ -159,5 +159,7 @@ "For example, if you want to search for publications on Computing in 2018, excluding master thesis: go to the ": "Por exemplo, se você deseja buscar publicações sobre 'Computação' em 2018, excluindo dissertações: acesse a página do índice de ", "index page and use the query below": "e use a query abaixo", "See below for the full list of search fields that can be used for the index of": "Veja abaixo a lista completa dos campos de pesquisa que podem ser usados para o índice de", - "(all:Computing) AND (publicationDate:2018) AND NOT (type:master thesis)": "(all:Computação) AND (publicationDate:2018) AND NOT (type:master thesis)" + "(all:Computing) AND (publicationDate:2018) AND NOT (type:master thesis)": "(all:Computação) AND (publicationDate:2018) AND NOT (type:master thesis)", + "Download": "Baixar", + "Export csv": "Exportar para csv" } diff --git a/src/components/AllIndexVisNetwork.tsx b/src/components/AllIndexVisNetwork.tsx index e6c3369..6092000 100644 --- a/src/components/AllIndexVisNetwork.tsx +++ b/src/components/AllIndexVisNetwork.tsx @@ -231,7 +231,6 @@ function VisGraph() { }, []); useEffect(() => { - console.log('indexStat', indexesStats); const newNodes: IndexNode[] = []; const maxSizeOfNode = Math.max(...indexesStats.map((item) => item['docs.count'])); for (let i = 0; i < keysLanguage.length; i++) { diff --git a/src/components/BasicSearchBox.tsx b/src/components/BasicSearchBox.tsx index 4095047..caedd29 100644 --- a/src/components/BasicSearchBox.tsx +++ b/src/components/BasicSearchBox.tsx @@ -27,7 +27,6 @@ const BasicSearchBox = ({ const { t } = useTranslation('common'); const router = useRouter(); const [docsCount, setDocsCount] = useState(localStorage.getItem(indexName)); - console.log('indexName', indexName); useEffect(() => { ElasticSearchStatsService(indexName) diff --git a/src/components/DownloadModal.tsx b/src/components/DownloadModal.tsx new file mode 100644 index 0000000..325c0eb --- /dev/null +++ b/src/components/DownloadModal.tsx @@ -0,0 +1,79 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { SearchContext, withSearch } from '@elastic/react-search-ui'; +import { useTranslation } from 'next-i18next'; +import { useContext, useState } from 'react'; +import { Button, Modal } from 'react-bootstrap'; +import { FaFileExport } from 'react-icons/fa6'; +import BulkDownloadService from '../services/BulkDownloadService'; +import { CustomSearchQuery } from '../types/Entities'; +import Loader from './Loader'; +import { formatedQuery } from './indicators/query/Query'; + +type DownloadModalProps = { + filters?: any; + searchTerm?: any; +}; + +const DownloadModal = ({ filters, searchTerm }: DownloadModalProps) => { + const { t } = useTranslation('common'); + const { driver } = useContext(SearchContext); + const [show, setShow] = useState(false); + const [downloadLink, setDownloadLink] = useState(''); + const [isLoading, setLoading] = useState(false); + const handleClose = () => setShow(false); + const handleShow = () => setShow(true); + const { search_fields, operator, index } = driver.searchQuery as CustomSearchQuery; + // @ts-ignore + const fields = Object.keys(search_fields); + + async function handleDownload() { + try { + setLoading(true); + const query: QueryDslQueryContainer = formatedQuery(searchTerm, fields, operator, filters); + const response = await new BulkDownloadService().search(index, query); + const { file } = response; + console.log('download fim', file); + const nextDownloadLink = `/api/download?fileName=${file}`; + setDownloadLink(nextDownloadLink); + handleShow(); + } finally { + setLoading(false); + } + } + + return ( + <> + + {isLoading ? : ''} + + + {t('Download')} + + + + {t('Export csv')} + + + + + + + + ); +}; + +export default withSearch(({ filters, searchTerm }) => ({ + filters, + searchTerm, +}))(DownloadModal); diff --git a/src/components/indicators/GroupsIndicators.tsx b/src/components/indicators/GroupsIndicators.tsx index 79284f4..594e8fe 100644 --- a/src/components/indicators/GroupsIndicators.tsx +++ b/src/components/indicators/GroupsIndicators.tsx @@ -16,7 +16,7 @@ import { CustomSearchQuery, IndicatorType } from '../../types/Entities'; import { IndicatorsProps } from '../../types/Propos'; import IndicatorContext from '../context/CustomContext'; import { OptionsBar, OptionsPie } from './options/ChartsOptions'; -import getFormatedQuery from './query/Query'; +import { getAggregateQuery } from './query/Query'; ChartJS.register(CategoryScale, LinearScale, BarElement, Title, Tooltip, Legend, ArcElement); const INDEX_NAME = process.env.INDEX_GROUP || ''; @@ -67,7 +67,7 @@ function GroupsIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) { ? ElasticSearchService( [ JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'creationYear', searchTerm, @@ -78,7 +78,7 @@ function GroupsIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) { }) ), JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'researchLine', searchTerm, @@ -88,7 +88,7 @@ function GroupsIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) { }) ), JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'knowledgeArea', searchTerm, @@ -98,7 +98,7 @@ function GroupsIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) { }) ), JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'status', searchTerm, diff --git a/src/components/indicators/JornalsIndicators.tsx b/src/components/indicators/JornalsIndicators.tsx index 913eadc..de357ec 100644 --- a/src/components/indicators/JornalsIndicators.tsx +++ b/src/components/indicators/JornalsIndicators.tsx @@ -16,7 +16,7 @@ import { CustomSearchQuery, IndicatorType } from '../../types/Entities'; import { IndicatorsProps } from '../../types/Propos'; import IndicatorContext from '../context/CustomContext'; import { OptionsBar } from './options/ChartsOptions'; -import getFormatedQuery from './query/Query'; +import { getAggregateQuery } from './query/Query'; ChartJS.register(CategoryScale, LinearScale, BarElement, Title, Tooltip, Legend, ArcElement); const INDEX_NAME = process.env.INDEX_JOURNAL || ''; @@ -45,7 +45,7 @@ function JornalsIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) ? ElasticSearchService( [ JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'qualis', searchTerm, diff --git a/src/components/indicators/OrgUnitIndicators.tsx b/src/components/indicators/OrgUnitIndicators.tsx index 88e7e9d..428a3a8 100644 --- a/src/components/indicators/OrgUnitIndicators.tsx +++ b/src/components/indicators/OrgUnitIndicators.tsx @@ -16,7 +16,7 @@ import { CustomSearchQuery, IndicatorType } from '../../types/Entities'; import { IndicatorsProps } from '../../types/Propos'; import IndicatorContext from '../context/CustomContext'; import { OptionsBar } from './options/ChartsOptions'; -import getFormatedQuery from './query/Query'; +import { getAggregateQuery } from './query/Query'; ChartJS.register(CategoryScale, LinearScale, BarElement, Title, Tooltip, Legend, ArcElement); const INDEX_NAME = process.env.INDEX_ORGUNIT || ''; @@ -51,7 +51,7 @@ function OrgUnitIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) optionsState.plugins.title.text = t(optionsState.title); const countryQuery = JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'country', searchTerm, @@ -61,7 +61,7 @@ function OrgUnitIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) }) ); const stateQuery = JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'state', searchTerm, diff --git a/src/components/indicators/PatentsIndicators.tsx b/src/components/indicators/PatentsIndicators.tsx index a95022d..1803d71 100644 --- a/src/components/indicators/PatentsIndicators.tsx +++ b/src/components/indicators/PatentsIndicators.tsx @@ -16,7 +16,7 @@ import { CustomSearchQuery, IndicatorType } from '../../types/Entities'; import { IndicatorsProps } from '../../types/Propos'; import IndicatorContext from '../context/CustomContext'; import { OptionsBar, OptionsPie } from './options/ChartsOptions'; -import getFormatedQuery from './query/Query'; +import { getAggregateQuery } from './query/Query'; ChartJS.register(CategoryScale, LinearScale, BarElement, Title, Tooltip, Legend, ArcElement); const INDEX_NAME = process.env.INDEX_PATENT || ''; @@ -69,7 +69,7 @@ function PatentsIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) ? ElasticSearchService( [ JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'depositDate', searchTerm, @@ -80,7 +80,7 @@ function PatentsIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) }) ), JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'publicationDate', searchTerm, @@ -91,7 +91,7 @@ function PatentsIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) }) ), JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'countryCode', searchTerm, @@ -101,7 +101,7 @@ function PatentsIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) }) ), JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'kindCode', searchTerm, diff --git a/src/components/indicators/PeopleIndicators.tsx b/src/components/indicators/PeopleIndicators.tsx index ef8554e..f6dcf0a 100644 --- a/src/components/indicators/PeopleIndicators.tsx +++ b/src/components/indicators/PeopleIndicators.tsx @@ -19,7 +19,7 @@ import { CustomSearchQuery, IndicatorType } from '../../types/Entities'; import { IndicatorsProps } from '../../types/Propos'; import IndicatorContext from '../context/CustomContext'; import { OptionsPie } from './options/ChartsOptions'; -import getFormatedQuery from './query/Query'; +import { getAggregateQuery } from './query/Query'; ChartJS.register(CategoryScale, LinearScale, BarElement, Title, Tooltip, Legend, ArcElement); const INDEX_NAME = process.env.INDEX_PERSON || ''; @@ -38,7 +38,6 @@ const headersResearchArea = [ function PeopleIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) { const { t } = useTranslation('common'); - const { driver } = useContext(SearchContext); const { indicators, setIndicatorsData, isEmpty } = useContext(IndicatorContext); @@ -54,7 +53,7 @@ function PeopleIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) { ? ElasticSearchService( [ JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'nationality', searchTerm, @@ -64,7 +63,7 @@ function PeopleIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) { }) ), JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'researchArea', searchTerm, diff --git a/src/components/indicators/ProgramsIndicators.tsx b/src/components/indicators/ProgramsIndicators.tsx index 771bd20..2c1fff9 100644 --- a/src/components/indicators/ProgramsIndicators.tsx +++ b/src/components/indicators/ProgramsIndicators.tsx @@ -16,7 +16,7 @@ import { CustomSearchQuery, IndicatorType } from '../../types/Entities'; import { IndicatorsProps } from '../../types/Propos'; import IndicatorContext from '../context/CustomContext'; import { OptionsBar } from './options/ChartsOptions'; -import getFormatedQuery from './query/Query'; +import { getAggregateQuery } from './query/Query'; ChartJS.register(CategoryScale, LinearScale, BarElement, Title, Tooltip, Legend, ArcElement); const INDEX_NAME = process.env.INDEX_PROGRAM || ''; @@ -45,7 +45,7 @@ function ProgramsIndicators({ filters, searchTerm, isLoading }: IndicatorsProps) ? ElasticSearchService( [ JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'orgunit.name', searchTerm, diff --git a/src/components/indicators/PublicationsIndicators.tsx b/src/components/indicators/PublicationsIndicators.tsx index 8684656..d06e103 100644 --- a/src/components/indicators/PublicationsIndicators.tsx +++ b/src/components/indicators/PublicationsIndicators.tsx @@ -16,7 +16,7 @@ import { CustomSearchQuery, IndicatorType } from '../../types/Entities'; import { IndicatorsProps } from '../../types/Propos'; import IndicatorContext from '../context/CustomContext'; import { OptionsBar, OptionsPie } from './options/ChartsOptions'; -import getFormatedQuery from './query/Query'; +import { getAggregateQuery } from './query/Query'; ChartJS.register(CategoryScale, LinearScale, BarElement, Title, Tooltip, Legend, ArcElement); const INDEX_NAME = process.env.INDEX_PUBLICATION || ''; @@ -50,7 +50,7 @@ function PublicationsIndicators({ filters, searchTerm, isLoading }: IndicatorsPr optionsType.plugins.title.text = t(optionsType.title); try { const pdQuery = JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'publicationDate', searchTerm, @@ -61,7 +61,7 @@ function PublicationsIndicators({ filters, searchTerm, isLoading }: IndicatorsPr }) ); const typeQuery = JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'type', searchTerm, diff --git a/src/components/indicators/SoftwaresIndicators.tsx b/src/components/indicators/SoftwaresIndicators.tsx index c2ad512..3d55565 100644 --- a/src/components/indicators/SoftwaresIndicators.tsx +++ b/src/components/indicators/SoftwaresIndicators.tsx @@ -16,7 +16,7 @@ import { CustomSearchQuery, IndicatorType } from '../../types/Entities'; import { IndicatorsProps } from '../../types/Propos'; import IndicatorContext from '../context/CustomContext'; import { OptionsBar, OptionsPie } from './options/ChartsOptions'; -import getFormatedQuery from './query/Query'; +import { getAggregateQuery } from './query/Query'; ChartJS.register(CategoryScale, LinearScale, BarElement, Title, Tooltip, Legend, ArcElement); const INDEX_NAME = process.env.INDEX_SOFTWARE || ''; @@ -54,7 +54,7 @@ function SoftwaresIndicators({ filters, searchTerm, isLoading }: IndicatorsProps ? ElasticSearchService( [ JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'releaseYear', searchTerm, @@ -65,7 +65,7 @@ function SoftwaresIndicators({ filters, searchTerm, isLoading }: IndicatorsProps }) ), JSON.stringify( - getFormatedQuery({ + getAggregateQuery({ size: 10, indicadorName: 'knowledgeAreas', searchTerm, diff --git a/src/components/indicators/query/Query.tsx b/src/components/indicators/query/Query.tsx index 63becba..1d97992 100644 --- a/src/components/indicators/query/Query.tsx +++ b/src/components/indicators/query/Query.tsx @@ -8,10 +8,10 @@ type QueryProps = { searchTerm: string; fields: string[]; operator: QueryDslOperator; - filters: []; + filters: Filter[]; order?: { _count?: string; _key?: string }; }; -export default function getFormatedQuery({ +export function getAggregateQuery({ size = 10, indicadorName, searchTerm, @@ -21,24 +21,7 @@ export default function getFormatedQuery({ order = { _count: 'desc' }, }: QueryProps) { try { - let query: QueryDslQueryContainer = {}; - if (searchTerm.indexOf('(') >= 0) { - query = new QueryFormat().toElasticsearch(searchTerm, fields); - } else { - query = { - bool: { - must: { - query_string: { - query: searchTerm || '*', - default_operator: operator, - fields: searchTerm ? fields : [], - }, - }, - filter: filters && filters.length > 0 ? getFormatedFilters(filters) : [], - // minimum_should_match: 1, - }, - }; - } + const query: QueryDslQueryContainer = formatedQuery(searchTerm, fields, operator, filters); return { track_total_hits: true, _source: [indicadorName], @@ -59,6 +42,32 @@ export default function getFormatedQuery({ } } +export function formatedQuery( + searchTerm: string, + fields: string[], + operator: QueryDslOperator, + filters: Filter[] +): QueryDslQueryContainer { + let query: QueryDslQueryContainer = {}; + if (searchTerm.indexOf('(') >= 0) { + query = new QueryFormat().toElasticsearch(searchTerm, fields); + } else { + query = { + bool: { + must: { + query_string: { + query: searchTerm || '*', + default_operator: operator, + fields: searchTerm ? fields : [], + }, + }, + filter: filters && filters.length > 0 ? getFormatedFilters(filters) : [], + }, + }; + } + return query; +} + function getFormatedFilters(filters: Filter[]): any { const filterFormated: { terms: { [x: string]: FilterValue[] } }[] = []; filters.forEach((filter: Filter) => { diff --git a/src/pages/api/bulk-download.ts b/src/pages/api/bulk-download.ts new file mode 100644 index 0000000..c21afd0 --- /dev/null +++ b/src/pages/api/bulk-download.ts @@ -0,0 +1,103 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +import crypto from 'crypto'; +import { Client } from 'es7'; +import { Search } from 'es7/api/requestParams'; +import fs from 'fs'; +import { Json2CsvOptions, json2csv } from 'json-2-csv'; +import { NextApiRequest, NextApiResponse } from 'next'; + +// https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/7.17/scroll_examples.html + +const client = new Client({ + maxRetries: 5, + requestTimeout: 60000, + sniffOnStart: true, + node: process.env.HOST_ELASTIC, + auth: { + apiKey: process.env.API_KEY!, + }, +}); + +const proxy = async (req: NextApiRequest, res: NextApiResponse) => { + const { query, index } = req.body; + + let writeStream; + try { + const params: Search = { + index: index, + scroll: '30s', + size: 10000, + // _source: ['name'], + body: { + query: query, + }, + }; + + const fileName = getFileName(index, JSON.stringify(query)); + const path = `${process.env.DOWNLOAD_FOLDER_PATH}/${fileName}.csv`; + const createStream = fs.createWriteStream(path); + createStream.end(); + + writeStream = fs.createWriteStream(path); + + const options: Json2CsvOptions = { + prependHeader: true, + delimiter: { field: ';' }, + }; + + let isFirstItem = true; + let c = 1; + + for await (const hit of scrollSearch(params)) { + options.prependHeader = isFirstItem; + writeStream.write(json2csv(hit._source, options)); + writeStream.write('\r\n'); + isFirstItem = false; + console.log(c, JSON.stringify(params)); + c++; + } + console.log('params', JSON.stringify(params)); + res.json({ file: path }); + } catch (err) { + console.error('ERROR::', err); + res.status(400).json({ error: err.message }); + } finally { + console.log('close writeStream'); + writeStream?.end(); + } +}; + +// Scroll utility +async function* scrollSearch(params: Search) { + let response = await client.search(params); + + while (true) { + const sourceHits = response.body.hits.hits; + + if (sourceHits.length === 0) { + break; + } + + for (const hit of sourceHits) { + yield hit; + } + + if (!response.body._scroll_id) { + break; + } + + response = await client.scroll({ + scroll_id: response.body._scroll_id, + scroll: params.scroll, + }); + } +} + +function getFileName(index: string, query: string) { + const string = index + query; + const hash = crypto.createHash('sha256').update(string).digest('hex'); + return hash; +} + +export default proxy; diff --git a/src/pages/api/download.ts b/src/pages/api/download.ts new file mode 100644 index 0000000..7df9115 --- /dev/null +++ b/src/pages/api/download.ts @@ -0,0 +1,14 @@ +// Next.js API route support: https://nextjs.org/docs/api-routes/introduction +import fs from 'fs'; +import type { NextApiRequest, NextApiResponse } from 'next'; + +export default function handler(req: NextApiRequest, res: NextApiResponse) { + const fileName: string = (req.query.fileName as string) || (req.query.filename as string); + console.log('fileName', fileName); + const csvFile = fs.readFileSync(fileName, 'utf-8'); + res + .status(200) + .setHeader('Content-Type', 'text/csv') + .setHeader('Content-Disposition', `attachment; filename=${fileName}`) + .send(csvFile); +} diff --git a/src/pages/api/search.ts b/src/pages/api/search.ts index db29fc6..300a3df 100644 --- a/src/pages/api/search.ts +++ b/src/pages/api/search.ts @@ -35,7 +35,6 @@ function builConnector(index: string) { }, }; } - console.log('requestBody: ', JSON.stringify(requestBody.query)); return requestBody; } ); diff --git a/src/pages/groups.tsx b/src/pages/groups.tsx index 0feb325..21f3695 100644 --- a/src/pages/groups.tsx +++ b/src/pages/groups.tsx @@ -24,10 +24,12 @@ import { useState } from 'react'; import { containsResults } from '../../utils/Utils'; import CustomSearchBox from '../components/CustomSearchBox'; import DefaultQueryConfig from '../components/DefaultQueryConfig'; +import Loader from '../components/Loader'; import { CustomProvider } from '../components/context/CustomContext'; import CustomResultViewGroups from '../components/customResultView/CustomResultViewGroups'; import CustomViewPagingInfo from '../components/customResultView/CustomViewPagingInfo'; import GroupsIndicators from '../components/indicators/GroupsIndicators'; +import { CustomSearchDriverOptions } from '../types/Entities'; type Props = { // Add custom props here }; @@ -39,9 +41,10 @@ export const getStaticProps: GetStaticProps = async ({ locale }) => ({ }); const INDEX_NAME = process.env.INDEX_GROUP || ''; -const configDefault = { +const configDefault: CustomSearchDriverOptions = { ...DefaultQueryConfig(INDEX_NAME), searchQuery: { + index: INDEX_NAME, operator: 'OR', search_fields: { name_text: { @@ -176,8 +179,10 @@ export default function App() {
- ({ wasSearched, results })}> - {({ wasSearched, results }) => { + ({ wasSearched, results, isLoading })} + > + {({ wasSearched, results, isLoading }) => { return (
@@ -188,6 +193,7 @@ export default function App() {
+ {isLoading ? : ''} = async ({ locale }) => ({ }); const INDEX_NAME = process.env.INDEX_ORGUNIT || ''; -const configDefault = { +const configDefault: CustomSearchDriverOptions = { ...DefaultQueryConfig(INDEX_NAME), searchQuery: { + index: INDEX_NAME, operator: 'OR', search_fields: { name_text: {}, @@ -139,6 +142,7 @@ export default function App() { const [config, setConfig] = useState(configDefault); function updateOpetatorConfig(op: string) { + //@ts-ignore setConfig({ ...config, searchQuery: { ...config.searchQuery, operator: op } }); } @@ -150,8 +154,10 @@ export default function App() {
- ({ wasSearched, results })}> - {({ wasSearched, results }) => { + ({ wasSearched, results, isLoading })} + > + {({ wasSearched, results, isLoading }) => { return (
@@ -162,6 +168,7 @@ export default function App() {
+ {isLoading ? : ''} = async ({ locale }) => ({ const INDEX_NAME = process.env.INDEX_JOURNAL || ''; -const configDefault = { +const configDefault: CustomSearchDriverOptions = { ...DefaultQueryConfig(INDEX_NAME), searchQuery: { + index: INDEX_NAME, operator: 'OR', search_fields: { title_text: {}, @@ -163,6 +166,7 @@ export default function App() { const [config, setConfig] = useState(configDefault); function updateOpetatorConfig(op: string) { + //@ts-ignore setConfig({ ...config, searchQuery: { ...config.searchQuery, operator: op } }); } @@ -174,8 +178,10 @@ export default function App() {
- ({ wasSearched, results })}> - {({ wasSearched, results }) => { + ({ wasSearched, results, isLoading })} + > + {({ wasSearched, results, isLoading }) => { return (
@@ -186,6 +192,7 @@ export default function App() {
+ {isLoading ? : ''} = async ({ locale }) => ({ const INDEX_NAME = process.env.INDEX_PATENT || ''; -const configDefault = { +const configDefault: CustomSearchDriverOptions = { ...DefaultQueryConfig(INDEX_NAME), searchQuery: { + index: INDEX_NAME, operator: 'OR', search_fields: { espacenetTitle_text: {}, @@ -154,6 +157,7 @@ export default function App() { const [config, setConfig] = useState(configDefault); function updateOpetatorConfig(op: string) { + //@ts-ignore setConfig({ ...config, searchQuery: { ...config.searchQuery, operator: op } }); } @@ -165,8 +169,10 @@ export default function App() {
- ({ wasSearched, results })}> - {({ wasSearched, results }) => { + ({ wasSearched, results, isLoading })} + > + {({ wasSearched, results, isLoading }) => { return (
@@ -177,6 +183,7 @@ export default function App() {
+ {isLoading ? : ''} = async ({ locale }) => ({ }); const INDEX_NAME = process.env.INDEX_PERSON || ''; -const configDefault = { +const configDefault: CustomSearchDriverOptions = { ...DefaultQueryConfig(INDEX_NAME), searchQuery: { operator: 'OR', + index: INDEX_NAME, search_fields: { name_text: {}, lattesId: {}, @@ -137,6 +142,7 @@ export default function App() { const [config, setConfig] = useState(configDefault); function updateOpetatorConfig(op: string) { + //@ts-ignore setConfig({ ...config, searchQuery: { ...config.searchQuery, operator: op } }); } @@ -148,8 +154,10 @@ export default function App() {
- ({ wasSearched, results })}> - {({ wasSearched, results }) => { + ({ wasSearched, results, isLoading })} + > + {({ wasSearched, results, isLoading }) => { return (
@@ -160,6 +168,7 @@ export default function App() {
+ {isLoading ? : ''} } @@ -214,7 +224,11 @@ export default function App() {
)} - {containsResults(wasSearched, results) && } + {containsResults(wasSearched, results) && ( +
+ +
+ )} } // bodyFooter={} diff --git a/src/pages/programs.tsx b/src/pages/programs.tsx index ea1810e..e4d48b1 100644 --- a/src/pages/programs.tsx +++ b/src/pages/programs.tsx @@ -22,11 +22,13 @@ import { useState } from 'react'; import { containsResults } from '../../utils/Utils'; import CustomSearchBox from '../components/CustomSearchBox'; import DefaultQueryConfig from '../components/DefaultQueryConfig'; +import Loader from '../components/Loader'; import { CustomProvider } from '../components/context/CustomContext'; import CustomResultViewPrograms from '../components/customResultView/CustomResultViewPrograms'; import CustomViewPagingInfo from '../components/customResultView/CustomViewPagingInfo'; import ProgramsIndicators from '../components/indicators/ProgramsIndicators'; import styles from '../styles/Home.module.css'; +import { CustomSearchDriverOptions } from '../types/Entities'; type Props = { // Add custom props here }; @@ -38,9 +40,10 @@ export const getStaticProps: GetStaticProps = async ({ locale }) => ({ }); const INDEX_NAME = process.env.INDEX_PROGRAM || ''; -const configDefault = { +const configDefault: CustomSearchDriverOptions = { ...DefaultQueryConfig(INDEX_NAME), searchQuery: { + index: INDEX_NAME, operator: 'OR', search_fields: { name_text: {}, @@ -138,6 +141,7 @@ export default function App() { const [config, setConfig] = useState(configDefault); function updateOpetatorConfig(op: string) { + //@ts-ignore setConfig({ ...config, searchQuery: { ...config.searchQuery, operator: op } }); } @@ -149,8 +153,10 @@ export default function App() {
- ({ wasSearched, results })}> - {({ wasSearched, results }) => { + ({ wasSearched, results, isLoading })} + > + {({ wasSearched, results, isLoading }) => { return (
@@ -161,6 +167,7 @@ export default function App() {
+ {isLoading ? : ''} = async ({ locale }) }); const INDEX_NAME = process.env.INDEX_PUBLICATION || ''; -console.log('INDEX_PUBLICATION: ', INDEX_NAME); const configDefault: CustomSearchDriverOptions = { ...DefaultQueryConfig(INDEX_NAME), searchQuery: { operator: 'OR', + index: INDEX_NAME, search_fields: { title_text: { weight: 3, @@ -219,8 +221,10 @@ export default function App() {
- ({ wasSearched, results })}> - {({ wasSearched, results }) => { + ({ wasSearched, results, isLoading })} + > + {({ wasSearched, results, isLoading }) => { return (
@@ -230,6 +234,7 @@ export default function App() {
+ {isLoading ? : ''}
)} - {containsResults(wasSearched, results) && } + + {containsResults(wasSearched, results) && ( +
+ +
+ )} } // bodyFooter={} diff --git a/src/pages/software.tsx b/src/pages/software.tsx index 5ce470c..c82ff0e 100644 --- a/src/pages/software.tsx +++ b/src/pages/software.tsx @@ -22,11 +22,13 @@ import { useState } from 'react'; import { containsResults } from '../../utils/Utils'; import CustomSearchBox from '../components/CustomSearchBox'; import DefaultQueryConfig from '../components/DefaultQueryConfig'; +import Loader from '../components/Loader'; import { CustomProvider } from '../components/context/CustomContext'; import CustomResultViewSoftwares from '../components/customResultView/CustomResultViewSoftwares'; import CustomViewPagingInfo from '../components/customResultView/CustomViewPagingInfo'; import SoftwaresIndicators from '../components/indicators/SoftwaresIndicators'; import styles from '../styles/Home.module.css'; +import { CustomSearchDriverOptions } from '../types/Entities'; type Props = { // Add custom props here }; @@ -38,9 +40,10 @@ export const getStaticProps: GetStaticProps = async ({ locale }) => ({ }); const INDEX_NAME = process.env.INDEX_SOFTWARE || ''; -const configDefault = { +const configDefault: CustomSearchDriverOptions = { ...DefaultQueryConfig(INDEX_NAME), searchQuery: { + index: INDEX_NAME, operator: 'OR', search_fields: { name_text: { @@ -166,6 +169,7 @@ export default function App() { const [config, setConfig] = useState(configDefault); function updateOpetatorConfig(op: string) { + //@ts-ignore setConfig({ ...config, searchQuery: { ...config.searchQuery, operator: op } }); } @@ -177,8 +181,10 @@ export default function App() {
- ({ wasSearched, results })}> - {({ wasSearched, results }) => { + ({ wasSearched, results, isLoading })} + > + {({ wasSearched, results, isLoading }) => { return (
@@ -189,6 +195,7 @@ export default function App() {
+ {isLoading ? : ''}