Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat/cms articles scraping #215 #501

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions .docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,13 @@ RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y

COPY --from=composer /usr/bin/composer /usr/bin/composer

#----------------------------------------------------------------------
# Installation de divers outils
#----------------------------------------------------------------------
RUN apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq install -y rclone git \
&& rclone self-update --package deb \
&& apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

#----------------------------------------------------------------------
# Nodejs : Installation
# https://github.com/nodejs/docker-node/blob/main/18/bullseye/Dockerfile
Expand All @@ -49,11 +56,6 @@ RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm
RUN npm i -g yarn pnpm npx
# RUN yarn set version stable

#----------------------------------------------------------------------
# Nettoyage Cache APT
#----------------------------------------------------------------------
RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

#----------------------------------------------------------------------
# Configurer apache
#----------------------------------------------------------------------
Expand All @@ -80,8 +82,7 @@ RUN export APP_ENV=prod \

RUN yarn install --production --frozen-lockfile \
&& yarn build \
&& yarn cache clean \
&& rm -rf node_modules
&& yarn cache clean

EXPOSE 8000

Expand Down
3 changes: 3 additions & 0 deletions .docker/cron/cron-articles-scraper.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/bin/bash

node bin/articles-scraper.mjs
6 changes: 6 additions & 0 deletions .eslintrc.json
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@
"rules": {
"react/jsx-uses-react": "error" // import React from 'react' est nécessaire seulement dans les fichiers javascript (non-typescript)
}
},
{
"files": ["bin/**"],
"env": {
"node": true
}
}
],
"settings": {
Expand Down
6 changes: 3 additions & 3 deletions assets/config/navItems.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { MainNavigationProps } from "@codegouvfr/react-dsfr/MainNavigation";
import { TranslationFunction } from "i18nifty/typeUtils/TranslationFunction";

import { ComponentKey, Translations, declareComponentKeys } from "../i18n/i18n";
import { routes } from "../router/router";
import { cartesUrl, catalogueUrl, routes } from "../router/router";
import { assistanceNavItems } from "./assistanceNavItems";

// dans ce cas précise, getTranslation ne marche pas parce que les traductions sont pas encore chargées, on est donc obglié de passer la fonction t en paramètre
Expand All @@ -27,11 +27,11 @@ export const defaultNavItems = (t: TranslationFunction<"navItems", ComponentKey>
},
{
text: t("catalog"),
linkProps: { href: "./catalogue" },
linkProps: { href: catalogueUrl },
},
{
text: t("maps"),
linkProps: { href: "./cartes" },
linkProps: { href: cartesUrl },
},
{
text: t("news"),
Expand Down
113 changes: 58 additions & 55 deletions assets/pages/news/NewsArticle.tsx
Original file line number Diff line number Diff line change
@@ -1,87 +1,90 @@
import { fr } from "@codegouvfr/react-dsfr";
import Tag from "@codegouvfr/react-dsfr/Tag";
import { FC } from "react";
import Alert from "@codegouvfr/react-dsfr/Alert";
import { useQuery } from "@tanstack/react-query";
import { FC, useMemo } from "react";
import { symToStr } from "tsafe/symToStr";

import { type NewsArticle } from "../../@types/newsArticle";
import AppLayout from "../../components/Layout/AppLayout";
import articles from "../../data/actualites.json";
import LoadingText from "../../components/Utils/LoadingText";
import { useTranslation } from "../../i18n/i18n";
import { appRoot, routes } from "../../router/router";
import { formatDateFromISO } from "../../utils";
import SymfonyRouting from "../../modules/Routing";
import { routes } from "../../router/router";
import PageNotFound from "../error/PageNotFound";

type NewsArticleProps = {
slug: string;
};

const NewsArticle: FC<NewsArticleProps> = ({ slug }) => {
const { t: tCommon } = useTranslation("Common");
const { t: tBreadcrumb } = useTranslation("Breadcrumb");

const newsArticle: NewsArticle | undefined = articles[slug];
const articleQuery = useQuery({
queryKey: ["articles", "slug", slug],
queryFn: async ({ signal }) => {
const url = SymfonyRouting.generate("cartesgouvfr_s3_gateway_get_content", {
path: `articles/${slug}.html`,
});
const response = await fetch(url, { signal });

if (newsArticle === undefined) {
if (!response.ok) {
return Promise.reject({
message: "Fetching articles failed",
code: response.status,
});
}

const text = await response.text();
return text;
},
});

const documentTitle = useMemo(() => {
if (articleQuery.data === undefined) return undefined;

const parser = new DOMParser();
const htmlDoc = parser.parseFromString(articleQuery.data, "text/html");

return htmlDoc.querySelector("h1")?.innerText;
}, [articleQuery.data]);

// @ts-expect-error fausse alerte
if (articleQuery.error?.code === 404) {
return <PageNotFound />;
}

const tags = newsArticle?.tags?.map((tag, i) => (
<Tag key={`${slug}_tag_${i}`} className={fr.cx("fr-mr-2v")}>
{tag}
</Tag>
));

return (
<AppLayout
documentTitle={newsArticle?.title}
documentTitle={documentTitle}
customBreadcrumbProps={{
homeLinkProps: routes.home().link,
segments: [{ label: tBreadcrumb("news"), linkProps: routes.news_list().link }],
currentPageLabel: newsArticle?.breadcrumb ?? newsArticle.title,
currentPageLabel: documentTitle,
}}
>
<div className={fr.cx("fr-grid-row")}>
<div className={fr.cx("fr-col-12", "fr-col-md-8")}>
<div className={fr.cx("fr-grid-row", "fr-grid-row--center")}>
<div className={fr.cx("fr-tags-group")}>{tags}</div>
</div>
<div className={fr.cx("fr-grid-row", "fr-grid-row--center")}>
<h1>{newsArticle?.title}</h1>
</div>
<div className={fr.cx("fr-grid-row", "fr-grid-row--center")}>
<p dangerouslySetInnerHTML={{ __html: newsArticle.short_description ?? "" }} />
</div>
<div className={fr.cx("fr-grid-row", "fr-grid-row--center")}>
<p
style={{
fontStyle: "italic",
color: fr.colors.decisions.text.mention.grey.default,
}}
>
<i className="ri-article-line" />
&nbsp;Publié le {formatDateFromISO(newsArticle.date)}
</p>
</div>
{articleQuery.isLoading && (
<div className={fr.cx("fr-container")}>
<LoadingText message="Actualités" as="h1" withSpinnerIcon={true} />
</div>
)}

<div className={fr.cx("fr-grid-row", "fr-grid-row--center")}>
<figure className={fr.cx("fr-content-media")} role="group" aria-label={newsArticle?.thumbnail_caption}>
<div className={fr.cx("fr-content-media__img")}>
<img
className={fr.cx("fr-responsive-img")}
src={`${appRoot}/${newsArticle.thumbnail_url}`}
alt={newsArticle?.thumbnail_alt}
role="presentation"
data-fr-js-ratio="true"
/>
</div>
<figcaption className={fr.cx("fr-content-media__caption")}>{newsArticle?.thumbnail_caption}</figcaption>
</figure>
</div>
{articleQuery.error && (
<div className={fr.cx("fr-container")}>
<Alert severity={"error"} title={tCommon("error")} description={articleQuery.error?.message} className={fr.cx("fr-my-3w")} />
</div>
)}

<div className={fr.cx("fr-grid-row", "fr-mt-2w")}>
<div className={fr.cx("fr-col")} dangerouslySetInnerHTML={{ __html: newsArticle.content }} />
</div>
{articleQuery.data && (
<div className={fr.cx("fr-grid-row")}>
<div
className={fr.cx("fr-col-12", "fr-col-md-8")}
dangerouslySetInnerHTML={{
__html: articleQuery.data,
}}
/>
</div>
</div>
)}
</AppLayout>
);
};
Expand Down
96 changes: 52 additions & 44 deletions assets/pages/news/NewsList.tsx
Original file line number Diff line number Diff line change
@@ -1,62 +1,70 @@
import { fr } from "@codegouvfr/react-dsfr";
import { Card } from "@codegouvfr/react-dsfr/Card";
import { Tag } from "@codegouvfr/react-dsfr/Tag";
import Alert from "@codegouvfr/react-dsfr/Alert";
import { useQuery } from "@tanstack/react-query";
import { FC } from "react";
import { symToStr } from "tsafe/symToStr";

import AppLayout from "../../components/Layout/AppLayout";
import articles from "../../data/actualites.json";
import { appRoot, routes } from "../../router/router";
import { type NewsArticle } from "../../@types/newsArticle";
import { formatDateFromISO } from "../../utils";
import LoadingText from "../../components/Utils/LoadingText";
import { useTranslation } from "../../i18n/i18n";
import SymfonyRouting from "../../modules/Routing";
import { routes } from "../../router/router";

type NewsListItemProps = {
slug: string;
newsArticle: NewsArticle;
// NOTE pour que la commande "react-dsfr update-icons" inclue l'icone article dans les assets qui est utilisée dans les articles
// fr-icon-article-line

type NewsListProps = {
page: number;
};
const NewsList: FC<NewsListProps> = ({ page = 0 }) => {
const { t: tCommon } = useTranslation("Common");

const NewsListItem: FC<NewsListItemProps> = ({ slug, newsArticle }) => {
const SHORT_DESC_MAX_CHAR = 120;
const articlesListQuery = useQuery({
queryKey: ["articles", "list", page],
queryFn: async ({ signal }) => {
const url = SymfonyRouting.generate("cartesgouvfr_s3_gateway_get_content", {
path: `articles/list/${page}.html`,
});
const response = await fetch(url, { signal });

const tags = newsArticle?.tags?.map((tag, i) => <Tag key={`${slug}_tag_${i}`}>{tag}</Tag>);
if (!response.ok) {
return Promise.reject({
message: "Fetching articles failed",
code: response.status,
});
}

return (
<div className={fr.cx("fr-col-sm-12", "fr-col-md-4", "fr-col-lg-4")}>
<Card
start={<div className={fr.cx("fr-tags-group")}>{tags}</div>}
desc={
<span
dangerouslySetInnerHTML={{
__html:
newsArticle?.short_description && newsArticle?.short_description.length > SHORT_DESC_MAX_CHAR
? newsArticle?.short_description.substring(0, 100) + "..."
: (newsArticle?.short_description ?? ""),
}}
/>
}
detail={newsArticle?.date && formatDateFromISO(newsArticle?.date)}
enlargeLink
imageAlt={newsArticle?.thumbnail_alt ?? "Vignette de l’article"}
imageUrl={`${appRoot}/${newsArticle.thumbnail_url}`}
linkProps={routes.news_article({ slug }).link}
title={<span dangerouslySetInnerHTML={{ __html: newsArticle?.title ?? "" }} />}
titleAs="h2"
/>
</div>
);
};
NewsListItem.displayName = symToStr({ NewsListItem });
const text = await response.text();
return text;
},
});

// @ts-expect-error fausse alerte
if (articlesListQuery.error?.code === 404) {
routes.news_list({ page: 0 }).replace();
}

const NewsList = () => {
return (
<AppLayout documentTitle="Actualités">
<div className={fr.cx("fr-container")}>
<h1>Actualités</h1>
{articlesListQuery.isLoading && (
<div className={fr.cx("fr-container")}>
<LoadingText message="Actualités" as="h1" withSpinnerIcon={true} />
</div>
)}

<div className={fr.cx("fr-grid-row", "fr-grid-row--gutters")}>
{Object.entries(articles)?.map(([slug, article]) => <NewsListItem key={slug} slug={slug} newsArticle={article} />)}
{articlesListQuery.error && (
<div className={fr.cx("fr-container")}>
<Alert severity={"error"} title={tCommon("error")} description={articlesListQuery.error?.message} className={fr.cx("fr-my-3w")} />
</div>
</div>
)}

{articlesListQuery.data && (
<div
dangerouslySetInnerHTML={{
__html: articlesListQuery.data,
}}
/>
)}
</AppLayout>
);
};
Expand Down
2 changes: 1 addition & 1 deletion assets/router/RouterRenderer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ const RouterRenderer: FC = () => {
case "contact_thanks":
return <Thanks />;
case "news_list":
return <NewsList />;
return <NewsList page={route.params.page} />;
case "news_article":
return <NewsArticle slug={route.params.slug} />;
case "faq":
Expand Down
10 changes: 8 additions & 2 deletions assets/router/router.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ import { createRouter, defineRoute, param } from "type-route";
import SymfonyRouting from "../modules/Routing";

export const appRoot = SymfonyRouting.getBaseUrl(); // (document.getElementById("root") as HTMLDivElement).dataset?.appRoot ?? "";
export const catalogueUrl = (document.getElementById("app_env") as HTMLDivElement)?.dataset?.["catalogueUrl"] ?? "/catalogue";
export const catalogueUrl = (document.getElementById("app_env") as HTMLDivElement)?.dataset?.["catalogueUrl"] ?? appRoot + "/catalogue";
export const cartesUrl = appRoot + "/cartes";

const routeDefs = {
// routes non protégées (doivent être listées plus bas dans publicRoutes)
Expand All @@ -18,7 +19,12 @@ const routeDefs = {
documentation: defineRoute(`${appRoot}/documentation`),
contact: defineRoute(`${appRoot}/nous-ecrire`),
contact_thanks: defineRoute(`${appRoot}/nous-ecrire/demande-envoyee`),
news_list: defineRoute(`${appRoot}/actualites`),
news_list: defineRoute(
{
page: param.query.optional.number.default(0),
},
() => `${appRoot}/actualites`
),
news_article: defineRoute(
{
slug: param.path.string,
Expand Down
Loading