diff --git a/README.md b/README.md index fbd8e6b..48e21d0 100644 --- a/README.md +++ b/README.md @@ -25,3 +25,28 @@ rye run mkdocs serve ``` The docs will be available at http://127.0.0.1:8000/ (Ctrl+left click to open). + +## Exporting markdown page as PDF + +### Install pandoc + +On Windows, it can be installed using package installer from main pandoc page: https://pandoc.org/installing.html. Alternatively, it can be installed with Chocolatey: +``` +choco install pandoc +choco install rsvg-convert miktex +``` +On MacOS: +``` +brew install pandoc +brew install librsvg homebrew/cask/basictex +``` + +### Basic usage + +``` +pandoc docs/{folder}/{file_name}.md -o {file_name}.pdf --toc --template=docs/assets/dyvenia-template.tex --listings +``` + +### Additional commands + +Sometimes pandoc is putting image on next page with text from next pargraphs laying above it. In such situation page break command below image will help: `\clearpage` diff --git a/docs/assets/dyvenia-template.tex b/docs/assets/dyvenia-template.tex new file mode 100644 index 0000000..411ecde --- /dev/null +++ b/docs/assets/dyvenia-template.tex @@ -0,0 +1,681 @@ +% Options for packages loaded elsewhere +\PassOptionsToPackage{unicode$for(hyperrefoptions)$,$hyperrefoptions$$endfor$}{hyperref} +\PassOptionsToPackage{hyphens}{url} +$if(colorlinks)$ +\PassOptionsToPackage{dvipsnames,svgnames,x11names}{xcolor} +$endif$ +$if(CJKmainfont)$ +\PassOptionsToPackage{space}{xeCJK} +$endif$ +% +\documentclass[ +$if(fontsize)$ + $fontsize$, +$endif$ +$if(papersize)$ + $papersize$paper, +$endif$ +$if(beamer)$ + ignorenonframetext, +$if(handout)$ + handout, +$endif$ +$if(aspectratio)$ + aspectratio=$aspectratio$, +$endif$ +$if(babel-lang)$ + $babel-lang$, +$endif$ +$endif$ +$for(classoption)$ + $classoption$$sep$, +$endfor$ +]{$documentclass$} +$if(beamer)$ +$if(background-image)$ +\usebackgroundtemplate{% + \includegraphics[width=\paperwidth]{$background-image$}% +} +% In beamer background-image does not work well when other images are used, so this is the workaround +\pgfdeclareimage[width=\paperwidth,height=\paperheight]{background}{$background-image$} +\usebackgroundtemplate{\pgfuseimage{background}} +$endif$ +\usepackage{pgfpages} +\setbeamertemplate{caption}[numbered] +\setbeamertemplate{caption label separator}{: } +\setbeamercolor{caption name}{fg=normal text.fg} +\beamertemplatenavigationsymbols$if(navigation)$$navigation$$else$empty$endif$ +$for(beameroption)$ +\setbeameroption{$beameroption$} +$endfor$ +% Prevent slide breaks in the middle of a paragraph +\widowpenalties 1 10000 +\raggedbottom +$if(section-titles)$ +\setbeamertemplate{part page}{ + \centering + \begin{beamercolorbox}[sep=16pt,center]{part title} + \usebeamerfont{part title}\insertpart\par + \end{beamercolorbox} +} +\setbeamertemplate{section page}{ + \centering + \begin{beamercolorbox}[sep=12pt,center]{section title} + \usebeamerfont{section title}\insertsection\par + \end{beamercolorbox} +} +\setbeamertemplate{subsection page}{ + \centering + \begin{beamercolorbox}[sep=8pt,center]{subsection title} + \usebeamerfont{subsection title}\insertsubsection\par + \end{beamercolorbox} +} +\AtBeginPart{ + \frame{\partpage} +} +\AtBeginSection{ + \ifbibliography + \else + \frame{\sectionpage} + \fi +} +\AtBeginSubsection{ + \frame{\subsectionpage} +} +$endif$ +$endif$ +$if(beamerarticle)$ +\usepackage{beamerarticle} % needs to be loaded first +$endif$ +\usepackage{amsmath,amssymb} +$if(linestretch)$ +\usepackage{setspace} +$endif$ +\usepackage{iftex} +\ifPDFTeX + \usepackage[$if(fontenc)$$fontenc$$else$T1$endif$]{fontenc} + \usepackage[utf8]{inputenc} + \usepackage{textcomp} % provide euro and other symbols +\else % if luatex or xetex +$if(mathspec)$ + \ifXeTeX + \usepackage{mathspec} % this also loads fontspec + \else + \usepackage{unicode-math} % this also loads fontspec + \fi +$else$ + \usepackage{unicode-math} % this also loads fontspec +$endif$ + \defaultfontfeatures{Scale=MatchLowercase}$-- must come before Beamer theme + \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1} +\fi +$if(fontfamily)$ +$else$ +$-- Set default font before Beamer theme so the theme can override it +\usepackage{lmodern} +$endif$ +$-- Set Beamer theme before user font settings so they can override theme +$if(beamer)$ +$if(theme)$ +\usetheme[$for(themeoptions)$$themeoptions$$sep$,$endfor$]{$theme$} +$endif$ +$if(colortheme)$ +\usecolortheme{$colortheme$} +$endif$ +$if(fonttheme)$ +\usefonttheme{$fonttheme$} +$endif$ +$if(mainfont)$ +\usefonttheme{serif} % use mainfont rather than sansfont for slide text +$endif$ +$if(innertheme)$ +\useinnertheme{$innertheme$} +$endif$ +$if(outertheme)$ +\useoutertheme{$outertheme$} +$endif$ +$endif$ +$-- User font settings (must come after default font and Beamer theme) +$if(fontfamily)$ +\usepackage[$for(fontfamilyoptions)$$fontfamilyoptions$$sep$,$endfor$]{$fontfamily$} +$endif$ +\ifPDFTeX\else + % xetex/luatex font selection +$if(mainfont)$ + $if(mainfontfallback)$ + \ifLuaTeX + \usepackage{luaotfload} + \directlua{luaotfload.add_fallback("mainfontfallback",{ + $for(mainfontfallback)$"$mainfontfallback$"$sep$,$endfor$ + })} + \fi + $endif$ + \setmainfont[$for(mainfontoptions)$$mainfontoptions$$sep$,$endfor$$if(mainfontfallback)$,RawFeature={fallback=mainfontfallback}$endif$]{$mainfont$} +$endif$ +$if(sansfont)$ + $if(sansfontfallback)$ + \ifLuaTeX + \usepackage{luaotfload} + \directlua{luaotfload.add_fallback("sansfontfallback",{ + $for(sansfontfallback)$"$sansfontfallback$"$sep$,$endfor$ + })} + \fi + $endif$ + \setsansfont[$for(sansfontoptions)$$sansfontoptions$$sep$,$endfor$$if(sansfontfallback)$,RawFeature={fallback=sansfontfallback}$endif$]{$sansfont$} +$endif$ +$if(monofont)$ + $if(monofontfallback)$ + \ifLuaTeX + \usepackage{luaotfload} + \directlua{luaotfload.add_fallback("monofontfallback",{ + $for(monofontfallback)$"$monofontfallback$"$sep$,$endfor$ + })} + \fi + $endif$ + \setmonofont[$for(monofontoptions)$$monofontoptions$$sep$,$endfor$$if(monofontfallback)$,RawFeature={fallback=monofontfallback}$endif$]{$monofont$} +$endif$ +$for(fontfamilies)$ + \newfontfamily{$fontfamilies.name$}[$for(fontfamilies.options)$$fontfamilies.options$$sep$,$endfor$]{$fontfamilies.font$} +$endfor$ +$if(mathfont)$ +$if(mathspec)$ + \ifXeTeX + \setmathfont(Digits,Latin,Greek)[$for(mathfontoptions)$$mathfontoptions$$sep$,$endfor$]{$mathfont$} + \else + \setmathfont[$for(mathfontoptions)$$mathfontoptions$$sep$,$endfor$]{$mathfont$} + \fi +$else$ + \setmathfont[$for(mathfontoptions)$$mathfontoptions$$sep$,$endfor$]{$mathfont$} +$endif$ +$endif$ +$if(CJKmainfont)$ + \ifXeTeX + \usepackage{xeCJK} + \setCJKmainfont[$for(CJKoptions)$$CJKoptions$$sep$,$endfor$]{$CJKmainfont$} + $if(CJKsansfont)$ + \setCJKsansfont[$for(CJKoptions)$$CJKoptions$$sep$,$endfor$]{$CJKsansfont$} + $endif$ + $if(CJKmonofont)$ + \setCJKmonofont[$for(CJKoptions)$$CJKoptions$$sep$,$endfor$]{$CJKmonofont$} + $endif$ + \fi +$endif$ +$if(luatexjapresetoptions)$ + \ifLuaTeX + \usepackage[$for(luatexjapresetoptions)$$luatexjapresetoptions$$sep$,$endfor$]{luatexja-preset} + \fi +$endif$ +$if(CJKmainfont)$ + \ifLuaTeX + \usepackage[$for(luatexjafontspecoptions)$$luatexjafontspecoptions$$sep$,$endfor$]{luatexja-fontspec} + \setmainjfont[$for(CJKoptions)$$CJKoptions$$sep$,$endfor$]{$CJKmainfont$} + \fi +$endif$ +\fi +$if(zero-width-non-joiner)$ +%% Support for zero-width non-joiner characters. +\makeatletter +\def\zerowidthnonjoiner{% + % Prevent ligatures and adjust kerning, but still support hyphenating. + \texorpdfstring{% + \TextOrMath{\nobreak\discretionary{-}{}{\kern.03em}% + \ifvmode\else\nobreak\hskip\z@skip\fi}{}% + }{}% +} +\makeatother +\ifPDFTeX + \DeclareUnicodeCharacter{200C}{\zerowidthnonjoiner} +\else + \catcode`^^^^200c=\active + \protected\def ^^^^200c{\zerowidthnonjoiner} +\fi +%% End of ZWNJ support +$endif$ +% Use upquote if available, for straight quotes in verbatim environments +\IfFileExists{upquote.sty}{\usepackage{upquote}}{} +\IfFileExists{microtype.sty}{% use microtype if available + \usepackage[$for(microtypeoptions)$$microtypeoptions$$sep$,$endfor$]{microtype} + \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts +}{} +$if(indent)$ +$else$ +\makeatletter +\@ifundefined{KOMAClassName}{% if non-KOMA class + \IfFileExists{parskip.sty}{% + \usepackage{parskip} + }{% else + \setlength{\parindent}{0pt} + \setlength{\parskip}{6pt plus 2pt minus 1pt}} +}{% if KOMA class + \KOMAoptions{parskip=half}} +\makeatother +$endif$ +$if(verbatim-in-note)$ +\usepackage{fancyvrb} +$endif$ +\usepackage{xcolor} +$if(geometry)$ +$if(beamer)$ +\geometry{$for(geometry)$$geometry$$sep$,$endfor$} +$else$ +\usepackage[$for(geometry)$$geometry$$sep$,$endfor$]{geometry} +$endif$ +$endif$ +\usepackage[top=1in, bottom=1in, left=0.75in, right=0.75in]{geometry} % dyvenia adjusted +$if(beamer)$ +\newif\ifbibliography +$endif$ +$if(listings)$ +\usepackage{listings} +\newcommand{\passthrough}[1]{#1} +\lstset{defaultdialect=[5.3]Lua} +\lstset{defaultdialect=[x86masm]Assembler} +$endif$ +$if(lhs)$ +\lstnewenvironment{code}{\lstset{language=Haskell,basicstyle=\small\ttfamily}}{} +$endif$ +$if(highlighting-macros)$ +$highlighting-macros$ +$endif$ +$if(tables)$ +\usepackage{longtable,booktabs,array} +$if(multirow)$ +\usepackage{multirow} +$endif$ +\usepackage{calc} % for calculating minipage widths +$if(beamer)$ +\usepackage{caption} +% Make caption package work with longtable +\makeatletter +\def\fnum@table{\tablename~\thetable} +\makeatother +$else$ +% Correct order of tables after \paragraph or \subparagraph +\usepackage{etoolbox} +\makeatletter +\patchcmd\longtable{\par}{\if@noskipsec\mbox{}\fi\par}{}{} +\makeatother +% Allow footnotes in longtable head/foot +\IfFileExists{footnotehyper.sty}{\usepackage{footnotehyper}}{\usepackage{footnote}} +\makesavenoteenv{longtable} +$endif$ +$endif$ +$if(graphics)$ +\usepackage{graphicx} +\makeatletter +\newsavebox\pandoc@box +\newcommand*\pandocbounded[1]{% scales image to fit in text height/width + \sbox\pandoc@box{#1}% + \Gscale@div\@tempa{\textheight}{\dimexpr\ht\pandoc@box+\dp\pandoc@box\relax}% + \Gscale@div\@tempb{\linewidth}{\wd\pandoc@box}% + \ifdim\@tempb\p@<\@tempa\p@\let\@tempa\@tempb\fi% select the smaller of both + \ifdim\@tempa\p@<\p@\scalebox{\@tempa}{\usebox\pandoc@box}% + \else\usebox{\pandoc@box}% + \fi% +} +% Set default figure placement to htbp +\def\fps@figure{htbp} +\makeatother +$endif$ +$if(svg)$ +\usepackage{svg} +$endif$ +$if(strikeout)$ +$-- also used for underline +\ifLuaTeX + \usepackage{luacolor} + \usepackage[soul]{lua-ul} +\else + \usepackage{soul} +$if(beamer)$ + \makeatletter + \let\HL\hl + \renewcommand\hl{% fix for beamer highlighting + \let\set@color\beamerorig@set@color + \let\reset@color\beamerorig@reset@color + \HL} + \makeatother +$endif$ +$if(CJKmainfont)$ + \ifXeTeX + % soul's \st doesn't work for CJK: + \usepackage{xeCJKfntef} + \renewcommand{\st}[1]{\sout{#1}} + \fi +$endif$ +\fi +$endif$ +\setlength{\emergencystretch}{3em} % prevent overfull lines +\providecommand{\tightlist}{% + \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}} +$if(numbersections)$ +\setcounter{secnumdepth}{$if(secnumdepth)$$secnumdepth$$else$5$endif$} +$else$ +\setcounter{secnumdepth}{-\maxdimen} % remove section numbering +$endif$ +$if(subfigure)$ +\usepackage{subcaption} +$endif$ +$if(beamer)$ +$else$ +$if(block-headings)$ +% Make \paragraph and \subparagraph free-standing +\makeatletter +\ifx\paragraph\undefined\else + \let\oldparagraph\paragraph + \renewcommand{\paragraph}{ + \@ifstar + \xxxParagraphStar + \xxxParagraphNoStar + } + \newcommand{\xxxParagraphStar}[1]{\oldparagraph*{#1}\mbox{}} + \newcommand{\xxxParagraphNoStar}[1]{\oldparagraph{#1}\mbox{}} +\fi +\ifx\subparagraph\undefined\else + \let\oldsubparagraph\subparagraph + \renewcommand{\subparagraph}{ + \@ifstar + \xxxSubParagraphStar + \xxxSubParagraphNoStar + } + \newcommand{\xxxSubParagraphStar}[1]{\oldsubparagraph*{#1}\mbox{}} + \newcommand{\xxxSubParagraphNoStar}[1]{\oldsubparagraph{#1}\mbox{}} +\fi +\makeatother +$endif$ +$endif$ +$if(pagestyle)$ +\pagestyle{$pagestyle$} +$endif$ +$if(csl-refs)$ +% definitions for citeproc citations +\NewDocumentCommand\citeproctext{}{} +\NewDocumentCommand\citeproc{mm}{% + \begingroup\def\citeproctext{#2}\cite{#1}\endgroup} +\makeatletter + % allow citations to break across lines + \let\@cite@ofmt\@firstofone + % avoid brackets around text for \cite: + \def\@biblabel#1{} + \def\@cite#1#2{{#1\if@tempswa , #2\fi}} +\makeatother +\newlength{\cslhangindent} +\setlength{\cslhangindent}{1.5em} +\newlength{\csllabelwidth} +\setlength{\csllabelwidth}{3em} +\newenvironment{CSLReferences}[2] % #1 hanging-indent, #2 entry-spacing + {\begin{list}{}{% + \setlength{\itemindent}{0pt} + \setlength{\leftmargin}{0pt} + \setlength{\parsep}{0pt} + % turn on hanging indent if param 1 is 1 + \ifodd #1 + \setlength{\leftmargin}{\cslhangindent} + \setlength{\itemindent}{-1\cslhangindent} + \fi + % set entry spacing + \setlength{\itemsep}{#2\baselineskip}}} + {\end{list}} +\usepackage{calc} +\newcommand{\CSLBlock}[1]{\hfill\break\parbox[t]{\linewidth}{\strut\ignorespaces#1\strut}} +\newcommand{\CSLLeftMargin}[1]{\parbox[t]{\csllabelwidth}{\strut#1\strut}} +\newcommand{\CSLRightInline}[1]{\parbox[t]{\linewidth - \csllabelwidth}{\strut#1\strut}} +\newcommand{\CSLIndent}[1]{\hspace{\cslhangindent}#1} +$endif$ +$if(lang)$ +\ifLuaTeX +\usepackage[bidi=basic]{babel} +\else +\usepackage[bidi=default]{babel} +\fi +$if(babel-lang)$ +\babelprovide[main,import]{$babel-lang$} +$if(mainfont)$ +\ifPDFTeX +\else +\babelfont{rm}[$for(mainfontoptions)$$mainfontoptions$$sep$,$endfor$$if(mainfontfallback)$,RawFeature={fallback=mainfontfallback}$endif$]{$mainfont$} +\fi +$endif$ +$endif$ +$for(babel-otherlangs)$ +\babelprovide[import]{$babel-otherlangs$} +$endfor$ +$for(babelfonts/pairs)$ +\babelfont[$babelfonts.key$]{rm}{$babelfonts.value$} +$endfor$ +% get rid of language-specific shorthands (see #6817): +\let\LanguageShortHands\languageshorthands +\def\languageshorthands#1{} +$if(selnolig-langs)$ +\ifLuaTeX + \usepackage[$for(selnolig-langs)$$it$$sep$,$endfor$]{selnolig} % disable illegal ligatures +\fi +$endif$ +$endif$ +$for(header-includes)$ +$header-includes$ +$endfor$ + +% dyvenia header/footer section start +\usepackage{fancyhdr} % Include the fancyhdr package to customize headers +\usepackage{graphicx} % Include the graphicx package to insert the image + +% Configure header +\pagestyle{fancy} % Activate fancy headers +\fancyhf{} % Clear all header and footer fields + +% Add logo to the header (left-aligned) +\fancyhead[L]{\includegraphics[width=3cm]{/Users/kwolski/dyvenia/home/data-excellence/docs/assets/images/dyvenia-logo-gruen-1.png}} % Adjust width and path as necessary + +% Optional: Add page number to footer +\fancyfoot[C]{\thepage} +% dyvenia header/footer section end + +% dyvenia code background start +\usepackage{listings} % Include the listings package for code blocks +\usepackage{xcolor} % Required for coloring + +\definecolor{customGray}{RGB}{232, 232, 232} + +% Define a custom style for code blocks +\lstset{ + backgroundcolor=\color{customGray}, + basicstyle=\ttfamily, + frame=single, + rulecolor=\color{black}, + tabsize=4, + breaklines=true, + showstringspaces=false +} +% dyvenia code background end + +$if(dir)$ +\ifPDFTeX + \TeXXeTstate=1 + \newcommand{\RL}[1]{\beginR #1\endR} + \newcommand{\LR}[1]{\beginL #1\endL} + \newenvironment{RTL}{\beginR}{\endR} + \newenvironment{LTR}{\beginL}{\endL} +\fi +$endif$ +$if(natbib)$ +\usepackage[$natbiboptions$]{natbib} +\bibliographystyle{$if(biblio-style)$$biblio-style$$else$plainnat$endif$} +$endif$ +$if(biblatex)$ +\usepackage[$if(biblio-style)$style=$biblio-style$,$endif$$for(biblatexoptions)$$biblatexoptions$$sep$,$endfor$]{biblatex} +$for(bibliography)$ +\addbibresource{$bibliography$} +$endfor$ +$endif$ +$if(nocite-ids)$ +\nocite{$for(nocite-ids)$$it$$sep$, $endfor$} +$endif$ +$if(csquotes)$ +\usepackage{csquotes} +$endif$ +\usepackage{bookmark} +\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available +\urlstyle{$if(urlstyle)$$urlstyle$$else$same$endif$} +$if(links-as-notes)$ +% Make links footnotes instead of hotlinks: +\DeclareRobustCommand{\href}[2]{#2\footnote{\url{#1}}} +$endif$ +$if(verbatim-in-note)$ +\VerbatimFootnotes % allow verbatim text in footnotes +$endif$ +\hypersetup{ +$if(title-meta)$ + pdftitle={$title-meta$}, +$endif$ +$if(author-meta)$ + pdfauthor={$author-meta$}, +$endif$ +$if(lang)$ + pdflang={$lang$}, +$endif$ +$if(subject)$ + pdfsubject={$subject$}, +$endif$ +$if(keywords)$ + pdfkeywords={$for(keywords)$$keywords$$sep$, $endfor$}, +$endif$ +$if(colorlinks)$ + colorlinks=true, + linkcolor={$if(linkcolor)$$linkcolor$$else$Maroon$endif$}, + filecolor={$if(filecolor)$$filecolor$$else$Maroon$endif$}, + citecolor={$if(citecolor)$$citecolor$$else$Blue$endif$}, + urlcolor={$if(urlcolor)$$urlcolor$$else$Blue$endif$}, +$else$ +$if(boxlinks)$ +$else$ + hidelinks, +$endif$ +$endif$ + pdfcreator={LaTeX via pandoc}} + +$if(title)$ +\title{$title$$if(thanks)$\thanks{$thanks$}$endif$} +$endif$ +$if(subtitle)$ +$if(beamer)$ +$else$ +\usepackage{etoolbox} +\makeatletter +\providecommand{\subtitle}[1]{% add subtitle to \maketitle + \apptocmd{\@title}{\par {\large #1 \par}}{}{} +} +\makeatother +$endif$ +\subtitle{$subtitle$} +$endif$ +\author{$for(author)$$author$$sep$ \and $endfor$} +\date{$date$} +$if(beamer)$ +$if(institute)$ +\institute{$for(institute)$$institute$$sep$ \and $endfor$} +$endif$ +$if(titlegraphic)$ +\titlegraphic{\includegraphics$if(titlegraphicoptions)$[$for(titlegraphicoptions)$$titlegraphicoptions$$sep$, $endfor$]$endif${$titlegraphic$}} +$endif$ +$if(logo)$ +\logo{\includegraphics{$logo$}} +$endif$ +$endif$ + +\begin{document} +$if(has-frontmatter)$ +\frontmatter +$endif$ +$if(title)$ +$if(beamer)$ +\frame{\titlepage} +$else$ +\maketitle +$endif$ +$if(abstract)$ +\begin{abstract} +$abstract$ +\end{abstract} +$endif$ +$endif$ + +$for(include-before)$ +$include-before$ + +$endfor$ +$if(toc)$ +$if(toc-title)$ +\renewcommand*\contentsname{$toc-title$} +$endif$ +$if(beamer)$ +\begin{frame}[allowframebreaks] +$if(toc-title)$ + \frametitle{$toc-title$} +$endif$ + \setcounter{tocdepth}{$toc-depth$} + \tableofcontents +\end{frame} +$else$ +{ +$if(colorlinks)$ +\hypersetup{linkcolor=$if(toccolor)$$toccolor$$else$$endif$} +$endif$ +\setcounter{tocdepth}{$toc-depth$} +\tableofcontents +} +$endif$ +$endif$ +$if(lof)$ +\listoffigures +$endif$ +$if(lot)$ +\listoftables +$endif$ +$if(linestretch)$ +\setstretch{$linestretch$} +$endif$ +$if(has-frontmatter)$ +\mainmatter +$endif$ +$body$ + +$if(has-frontmatter)$ +\backmatter +$endif$ +$if(natbib)$ +$if(bibliography)$ +$if(biblio-title)$ +$if(has-chapters)$ +\renewcommand\bibname{$biblio-title$} +$else$ +\renewcommand\refname{$biblio-title$} +$endif$ +$endif$ +$if(beamer)$ +\begin{frame}[allowframebreaks]{$biblio-title$} + \bibliographytrue +$endif$ + \bibliography{$for(bibliography)$$bibliography$$sep$,$endfor$} +$if(beamer)$ +\end{frame} +$endif$ + +$endif$ +$endif$ +$if(biblatex)$ +$if(beamer)$ +\begin{frame}[allowframebreaks]{$biblio-title$} + \bibliographytrue + \printbibliography[heading=none] +\end{frame} +$else$ +\printbibliography$if(biblio-title)$[title=$biblio-title$]$endif$ +$endif$ + +$endif$ +$for(include-after)$ +$include-after$ + +$endfor$ +\end{document} diff --git a/docs/assets/images/dyvenia-logo-gruen-1.png b/docs/assets/images/dyvenia-logo-gruen-1.png new file mode 100644 index 0000000..5a2619e Binary files /dev/null and b/docs/assets/images/dyvenia-logo-gruen-1.png differ diff --git a/docs/data_platform/lifecycle.md b/docs/data_platform/lifecycle.md index 08e9c67..5751795 100644 --- a/docs/data_platform/lifecycle.md +++ b/docs/data_platform/lifecycle.md @@ -9,44 +9,44 @@ The data engineering lifecycle encompasses the various stages of designing, buil The data engineering lifecycle is supported by several critical undercurrents, including security, data governance, DataOps, data architecture, orchestration, and software engineering practices. While these foundational concepts are integral to the success and scalability of data engineering workflows, they will not be covered in detail in this document. However, they remain crucial to our overall approach and should be considered throughout every stage of the lifecycle. -**Diagram Placeholder: Data Engineering Lifecycle to be replaced** - -![data-engineering-lifecycle](../assets/images/data-platform_data-engineering-lifecycle.png) +![Data Engineering Lifecycle](../assets/images/data-platform_data-engineering-lifecycle.png) --- -### 1. **Data Generation** +## 1. **Data Generation** Data generation is the first step in the lifecycle, where raw data is created or collected from various sources. These sources can include user activity on applications, sensor data from IoT devices, logs from systems, and external APIs. The goal of this phase is to collect raw, unprocessed data that will be used in downstream processes. -#### Key Activities: +### Key Activities: - **Event Logging**: Capturing user or system events. - **IoT and Sensor Data**: Collecting data from physical devices. - **API Requests**: Pulling data from external sources. - **Manual Input**: Data entered by users or operators. **Diagram Placeholder: Data Generation Process** -![data generation - application database](../assets/images/data-platform_data-generation-application-database.png) -![data generation - IoT swarm and message queue](../assets/images/data-platform_data-generation-iot-swarm-and-message-queue.png) + +![Data Generation - Application Database](../assets/images/data-platform_data-generation-application-database.png) + +![Data Generation - IoT Swarm and Message Queue](../assets/images/data-platform_data-generation-iot-swarm-and-message-queue.png) --- -### 2. **Data Storage** +## 2. **Data Storage** Data storage is the foundational component where all ingested and processed data is stored securely and durably. The choice of storage system depends on the volume, velocity, and type of data. Data can be stored in structured databases, semi-structured storage, or unstructured data lakes. -#### Key Activities: +### Key Activities: - **Choosing Storage Systems**: Selecting between data warehouses, lakes, and object storage. - **Partitioning and Indexing**: Organizing data for faster access. - **Security and Compliance**: Ensuring data is encrypted and stored in compliance with regulatory requirements. --- -### 3. **Data Ingestion** +## 3. **Data Ingestion** Data ingestion refers to the process of bringing generated data into your data infrastructure. This can be done in real-time or through batch processes. The goal is to move data from its source into a system where it can be stored and transformed. -#### Key Activities: +### Key Activities: - **Streaming Ingestion**: Handling real-time data (e.g., Kafka, Pulsar). - **Batch Ingestion**: Periodically importing data (e.g., using ETL tools). - **API or File-Based Ingestion**: Pulling data from external services or systems via APIs or file transfers. @@ -55,11 +55,11 @@ Data ingestion refers to the process of bringing generated data into your data i --- -### 4. **Data Transformation** +## 4. **Data Transformation** Data transformation is where the raw data is cleaned, structured, and enriched to make it suitable for analysis or other uses. This phase involves applying business logic, converting data formats, aggregating values, and removing noise. -#### Key Activities: +### Key Activities: - **Data Cleaning**: Handling missing, duplicate, or erroneous data. - **Data Enrichment**: Adding additional information or context to raw data. - **Aggregations and Calculations**: Summing, averaging, or transforming data based on business rules. @@ -67,11 +67,11 @@ Data transformation is where the raw data is cleaned, structured, and enriched t --- -### 5. **Data Serving** +## 5. **Data Serving** Once the data is transformed, it needs to be made available for use in real-time systems or analytical tools. Data serving is about making sure that transformed data is accessible and delivered to users, applications, or other systems in a timely and efficient manner. -#### Key Activities: +### Key Activities: - **API Exposure**: Making data available through APIs. - **Data Warehousing**: Storing processed data in data warehouses for efficient querying. - **Data Lakes**: Providing storage for large volumes of raw or semi-structured data. @@ -79,22 +79,22 @@ Once the data is transformed, it needs to be made available for use in real-time --- -### 6. **Analytics** +## 6. **Analytics** After data is stored and made available, it is used for analytics to extract insights. This stage typically involves querying data, building reports, dashboards, and conducting exploratory analysis. The insights derived here help inform business decisions. -#### Key Activities: +### Key Activities: - **BI Tools**: Generating reports and dashboards using tools like PowerBI, Tableau, or Looker. - **Exploratory Data Analysis**: Querying data to find patterns, trends, and insights. - **KPI Tracking**: Monitoring key performance indicators through dashboards. --- -### 7. **Machine Learning** +## 7. **Machine Learning** Machine learning is an optional stage in the data engineering lifecycle. Not all data pipelines will require machine learning, but for those that do, this stage is where cleaned and transformed data is used to train, validate, and deploy predictive models. Machine learning models help automate decision-making processes or provide forecasts based on historical data patterns. -#### Key Activities: +### Key Activities: - **Model Training**: Using historical data to train predictive models. - **Feature Engineering**: Creating new variables that better represent underlying patterns in the data. - **Model Deployment**: Putting machine learning models into production to make predictions in real time. @@ -102,16 +102,17 @@ Machine learning is an optional stage in the data engineering lifecycle. Not all --- -### 8. **Reverse ETL** +## 8. **Reverse ETL** Reverse ETL is the process of moving data from your centralized data warehouse or data lake back into operational systems (such as CRMs, marketing platforms, or custom applications). This stage is about operationalizing data by sending it back into the tools that teams use day-to-day. -#### Key Activities: +### Key Activities: - **Syncing Data**: Moving data back into operational systems like Salesforce, HubSpot, or custom tools. - **Operational Analytics**: Making transformed data actionable in real-time for business operations. - **Data Enrichment**: Enhancing third-party tools with enriched or aggregated data from internal systems. **Diagram Placeholder: Reverse ETL Process** + ![Reverse ETL](../assets/images/data-platform_reverse-etl.png) ---