Merge pull request #925 from basedosdados/staging

Staging
basedosdados · Aug 20, 2024 · 2066171 · 2066171
2 parents f911362 + e430d04
commit 2066171
Show file tree

Hide file tree

Showing 26 changed files with 815 additions and 410 deletions.
diff --git a/.github/workflows/release-dev.yaml b/.github/workflows/release-dev.yaml
@@ -36,6 +36,16 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Import Secrets
+        id: import_secrets
+        uses: hashicorp/[email protected]
+        with:
+          url: https://vault.basedosdados.org
+          token: ${{ secrets.VAULT_TOKEN }}
+          secrets: |
+            secret/data/url_download_data    URL_DOWNLOAD_CLOSED | URL_DOWNLOAD_CLOSED ;
+            secret/data/url_download_data    URL_DOWNLOAD_OPEN   | URL_DOWNLOAD_OPEN ;
+
       - name: Build and push
         uses: docker/build-push-action@v3
         with:
@@ -48,3 +58,6 @@ jobs:
             NEXT_PUBLIC_SITE_NAME=${{ env.NEXT_PUBLIC_SITE_NAME }}
             NEXT_PUBLIC_KEY_STRIPE=${{ env.NEXT_PUBLIC_KEY_STRIPE }}
             NEXT_PUBLIC_BASE_URL_FRONTEND=${{ env.NEXT_PUBLIC_BASE_URL_FRONTEND }}
+            URL_DOWNLOAD_CLOSED=${{ steps.import_secrets.outputs.URL_DOWNLOAD_CLOSED }}
+            URL_DOWNLOAD_OPEN=${{ steps.import_secrets.outputs.URL_DOWNLOAD_OPEN }}
+
diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml
@@ -36,6 +36,16 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Import Secrets
+        id: import_secrets
+        uses: hashicorp/[email protected]
+        with:
+          url: https://vault.basedosdados.org
+          token: ${{ secrets.VAULT_TOKEN }}
+          secrets: |
+            secret/data/url_download_data    URL_DOWNLOAD_CLOSED | URL_DOWNLOAD_CLOSED ;
+            secret/data/url_download_data    URL_DOWNLOAD_OPEN   | URL_DOWNLOAD_OPEN ;
+
       - name: Build and push
         uses: docker/build-push-action@v3
         with:
@@ -48,3 +58,5 @@ jobs:
             NEXT_PUBLIC_SITE_NAME=${{ env.NEXT_PUBLIC_SITE_NAME }}
             NEXT_PUBLIC_KEY_STRIPE=${{ env.NEXT_PUBLIC_KEY_STRIPE }}
             NEXT_PUBLIC_BASE_URL_FRONTEND=${{ env.NEXT_PUBLIC_BASE_URL_FRONTEND }}
+            URL_DOWNLOAD_CLOSED=${{ steps.import_secrets.outputs.URL_DOWNLOAD_CLOSED }}
+            URL_DOWNLOAD_OPEN=${{ steps.import_secrets.outputs.URL_DOWNLOAD_OPEN }}
diff --git a/.github/workflows/release-staging.yaml b/.github/workflows/release-staging.yaml
@@ -36,6 +36,16 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Import Secrets
+        id: import_secrets
+        uses: hashicorp/[email protected]
+        with:
+          url: https://vault.basedosdados.org
+          token: ${{ secrets.VAULT_TOKEN }}
+          secrets: |
+            secret/data/url_download_data    URL_DOWNLOAD_CLOSED | URL_DOWNLOAD_CLOSED ;
+            secret/data/url_download_data    URL_DOWNLOAD_OPEN   | URL_DOWNLOAD_OPEN ;
+
       - name: Build and push
         uses: docker/build-push-action@v3
         with:
@@ -48,3 +58,5 @@ jobs:
             NEXT_PUBLIC_SITE_NAME=${{ env.NEXT_PUBLIC_SITE_NAME }}
             NEXT_PUBLIC_KEY_STRIPE=${{ env.NEXT_PUBLIC_KEY_STRIPE }}
             NEXT_PUBLIC_BASE_URL_FRONTEND=${{ env.NEXT_PUBLIC_BASE_URL_FRONTEND }}
+            URL_DOWNLOAD_CLOSED=${{ steps.import_secrets.outputs.URL_DOWNLOAD_CLOSED }}
+            URL_DOWNLOAD_OPEN=${{ steps.import_secrets.outputs.URL_DOWNLOAD_OPEN }}
diff --git a/next/Dockerfile b/next/Dockerfile
@@ -12,14 +12,22 @@ RUN \
     else echo "Lockfile not found." && exit 1; \
     fi
 
+ARG URL_DOWNLOAD_CLOSED
+ARG URL_DOWNLOAD_OPEN
+
 # Rebuild the source code only when needed
 FROM node:18.17.0-alpine AS builder
+
 ARG NEXT_PUBLIC_API_URL
 ARG NEXT_PUBLIC_KEY_STRIPE
 ARG NEXT_PUBLIC_BASE_URL_FRONTEND
+ARG URL_DOWNLOAD_CLOSED
+ARG URL_DOWNLOAD_OPEN
+
 ENV NEXT_PUBLIC_API_URL=$NEXT_PUBLIC_API_URL
 ENV NEXT_PUBLIC_KEY_STRIPE=$NEXT_PUBLIC_KEY_STRIPE
 ENV NEXT_PUBLIC_BASE_URL_FRONTEND=$NEXT_PUBLIC_BASE_URL_FRONTEND
+
 WORKDIR /app
 COPY --from=deps /app/node_modules ./node_modules
 COPY . .
@@ -35,8 +43,13 @@ RUN npm run build
 FROM node:18.17.0-alpine AS runner
 WORKDIR /app
 
+ARG URL_DOWNLOAD_CLOSED
+ARG URL_DOWNLOAD_OPEN
+
 ENV NODE_ENV production
 ENV NODE_OPTIONS '-r next-logger'
+ENV URL_DOWNLOAD_CLOSED=$URL_DOWNLOAD_CLOSED
+ENV URL_DOWNLOAD_OPEN=$URL_DOWNLOAD_OPEN
 # Uncomment the following line in case you want to disable telemetry during runtime.
 # ENV NEXT_TELEMETRY_DISABLED 1
 

diff --git a/next/components/atoms/Toggle.js b/next/components/atoms/Toggle.js
@@ -1,10 +1,10 @@
 import { Switch } from "@chakra-ui/react";
 import styles from "../../styles/toggle.module.css";
 
-export default function Toggle({ value, onChange, ...props }) {
+export default function Toggle({ value, onChange, className = "toggle", ...props }) {
   return (
     <Switch
-      className={styles.toggle}
+      className={styles[className]}
       value={value}
       onChange={onChange}
       {...props}

diff --git a/next/components/molecules/ColumnsTable.js b/next/components/molecules/ColumnsTable.js
@@ -260,6 +260,7 @@ export default function ColumnsTable({
       if(result?.error) return
 
       let cloudTables = result?.cloudTables?.edges[0]?.node
+      const downloadInfo = HasDownloadPermitted(result?.uncompressedFileSize)
 
       triggerGAEvent("download_da_tabela",`{
         gcp: ${cloudTables?.gcpProjectId+"."+cloudTables?.gcpDatasetId+"."+cloudTables?.gcpTableId},
@@ -268,7 +269,8 @@ export default function ColumnsTable({
         table: ${resource?._id},
         columnDownload: true
       }`)
-      window.open(`https://storage.googleapis.com/basedosdados-public/one-click-download/${cloudTables?.gcpDatasetId}/${cloudTables?.gcpTableId}/${cloudTables?.gcpTableId}.csv.gz`)
+
+      window.open(`/api/tables/downloadTable?p=${btoa(cloudTables?.gcpDatasetId)}&q=${btoa(cloudTables?.gcpTableId)}&d=${btoa(downloadInfo.downloadPermitted)}&s=${btoa(downloadInfo.downloadWarning)}`)
     }
 
     return (
@@ -305,15 +307,13 @@ export default function ColumnsTable({
 
     const datasetName = value?.table?.dataset?.name || ""
     const tableName = value?.table?.name || ""
-  
+
     if(gcpDatasetID === "br_bd_diretorios_data_tempo") return "Não precisa de tradução"
     if(gcpDatasetID === "br_bd_diretorios_brasil") {
       if(gcpTableId === "empresa" || gcpTableId === "cep") return "Não precisa de tradução"
     }
     if(value?.name === "ddd") return "Não precisa de tradução"
 
-    const downloadUrl = `https://storage.googleapis.com/basedosdados-public/one-click-download/${gcpDatasetID}/${gcpTableId}/${gcpTableId}.csv.gz`
-
     return (
       <Box>
         {value === null ?
@@ -325,7 +325,10 @@ export default function ColumnsTable({
             <Text
               as="a"
               target="_blank"
-              href={downloadUrl}
+              href={value?.table?.isClosed || !downloadInfo.downloadPermitted
+                ? `${process.env.NEXT_PUBLIC_BASE_URL_FRONTEND}/dataset/${value?.table?.dataset?._id}?table=${value?.table?._id}`
+                : `/api/tables/downloadTable?p=${btoa(gcpDatasetID)}&q=${btoa(gcpTableId)}&d=${btoa(downloadInfo.downloadPermitted)}&s=${btoa(downloadInfo.downloadWarning)}`
+              }
               display="flex"
               onClick={() => {
                 if(!downloadInfo.downloadPermitted) return

diff --git a/next/components/molecules/DataInformationQuery.js b/next/components/molecules/DataInformationQuery.js
@@ -10,7 +10,9 @@ import {
   useClipboard,
   Tooltip,
   Skeleton,
-  Stack
+  Stack,
+  useDisclosure,
+  ModalCloseButton
 } from "@chakra-ui/react";
 import { useState, useEffect, useRef } from "react";
 import hljs from "highlight.js/lib/core";
@@ -23,6 +25,8 @@ import 'highlight.js/styles/obsidian.css'
 import GreenTab from "../atoms/GreenTab";
 import Toggle from "../atoms/Toggle";
 import ColumnsTable from "./ColumnsTable";
+import { SectionPrice } from "../../pages/precos";
+import { ModalGeneral } from "./uiUserPage";
 import { AlertDiscalimerBox} from "./DisclaimerBox";
 import { triggerGAEvent, formatBytes } from "../../utils";
 
@@ -171,7 +175,8 @@ export function CodeHighlight({ language, children }) {
 export default function DataInformationQuery({ resource }) {
   const [tabAccessIndex, setTabAccessIndex] = useState(0)
   const [tabIndex, setTabIndex] = useState(0)
-  const [downloadNotAllowed, setDownloadNotAllowed] = useState(false)
+  const [downloadPermitted, setDownloadPermitted] = useState(false)
+  const [downloadWarning, setDownloadWarning] = useState("")
   const [checkedColumns, setCheckedColumns] = useState([])
   const [numberColumns, setNumberColumns] = useState(0)
   const [columnsTranslationPro, setColumnsTranslationPro] = useState([])
@@ -181,11 +186,11 @@ export default function DataInformationQuery({ resource }) {
   const [hasLoadingColumns, setHasLoadingColumns] = useState(true)
   const [isLoadingCode, setIsLoadingCode] = useState(false)
   const [hasLoadingResponse, setHasLoadingResponse] = useState(false)
+  const plansModal = useDisclosure()
 
   const [gcpProjectID, setGcpProjectID] = useState("")
   const [gcpDatasetID, setGcpDatasetID] = useState("")
   const [gcpTableId, setGcpTableId] = useState("")
-  const [downloadUrl, setDownloadUrl] = useState("")
 
   const isUserPro = () => {
     let user
@@ -200,17 +205,27 @@ export default function DataInformationQuery({ resource }) {
   }, [resource.dataset])
 
   useEffect(() => {
-    if (resource?.numberRows === 0) setDownloadNotAllowed(false)
-    if (resource?.numberRows) resource?.numberRows > 200000 ? setDownloadNotAllowed(false) : setDownloadNotAllowed(true)
-
+    if (resource?.uncompressedFileSize) {
+      const limit100MB = 100 * 1024 * 1024;
+      const limit1GB = 1 * 1024 * 1024 * 1024;
+
+      if (resource?.uncompressedFileSize < limit100MB) {
+        setDownloadPermitted(true)
+        setDownloadWarning("free")
+      } else if (resource?.uncompressedFileSize < limit1GB) {
+        setDownloadPermitted(isUserPro())
+        setDownloadWarning("100mbBetween1gb")
+      } else {
+        setDownloadWarning("biggest1gb")
+      }
+    }
+
     if (resource?.cloudTables?.[0]) {
       setGcpProjectID(resource.cloudTables[0]?.gcpProjectId || "")
       setGcpDatasetID(resource.cloudTables[0]?.gcpDatasetId || "")
       setGcpTableId(resource.cloudTables[0]?.gcpTableId || "")
     }
-
-    setDownloadUrl(`https://storage.googleapis.com/basedosdados-public/one-click-download/${resource?.cloudTables?.[0]?.gcpDatasetId || ""}/${resource?.cloudTables?.[0]?.gcpTableId || ""}/${resource?.cloudTables?.[0]?.gcpTableId || ""}.csv.gz`)
-  }, [resource.numberRows, resource.cloudTables])
+  }, [resource.uncompressedFileSize, resource.cloudTables])
 
   useEffect(() => {
     if(resource._id === undefined) return
@@ -287,6 +302,36 @@ export default function DataInformationQuery({ resource }) {
       border="1px solid #DEDFE0"
       borderRadius="16px"
     >
+      <ModalGeneral
+        isOpen={plansModal.isOpen}
+        onClose={plansModal.onClose}
+        propsModalContent={{
+          minWidth: "fit-content"
+        }}
+      >
+        <Stack spacing={0} marginBottom="16px">
+          <Text
+            width="100%"
+            fontFamily="Roboto"
+            fontWeight="400"
+            color="#252A32"
+            fontSize="24px"
+            textAlign="center"
+            lineHeight="40px"
+          >
+            Compare os planos
+          </Text>
+          <ModalCloseButton
+            fontSize="14px"
+            top="34px"
+            right="26px"
+            _hover={{backgroundColor: "transparent", color:"#0B89E2"}}
+          />
+        </Stack>
+
+        <SectionPrice/>
+      </ModalGeneral>
+
       <Tabs
         width="100%"
         variant="unstyled"
@@ -512,8 +557,7 @@ export default function DataInformationQuery({ resource }) {
               marginTop="16px"
               padding={0}
             >
-              {downloadNotAllowed ?
-                isUserPro() ? "" :
+              {isUserPro() === false && downloadWarning === "free" &&
                 <AlertDiscalimerBox type="info">
                   Estes dados estão disponíveis porque diversas pessoas colaboram para a sua manutenção. <Text as="br" display={{base: "none", lg: "flex"}}/>
                   Antes de baixar os dados, apoie você também com uma doação financeira ou
@@ -528,41 +572,59 @@ export default function DataInformationQuery({ resource }) {
                       saiba como contribuir com seu tempo.
                     </Text>
                 </AlertDiscalimerBox>
-              :
+              }
+              {isUserPro() === false && downloadWarning === "100mbBetween1gb" &&
+                <AlertDiscalimerBox
+                  type="warning"
+                >
+                  O download de tabelas com tamanho entre 100 MB e 1 GB está disponível apenas para
+                  <Text
+                    marginLeft="4px"
+                    as="a"
+                    target="_blank"
+                    href="https://basedosdados.github.io/mais/colab_data/"
+                    color="#0068C5"
+                    _hover={{color: "#0057A4"}}
+                  >assinantes dos nossos planos pagos
+                  </Text>. No entanto, você pode acessar a tabela gratuitamente utilizando SQL, Python, R ou Stata. Considere atualizar para um plano pago para fazer o download.
+                </AlertDiscalimerBox>
+              }
+              {downloadWarning === "biggest1gb" &&
                 <AlertDiscalimerBox
                   type="error"
-                  text={`O tamanho da tabela ultrapassou o limite permitido para download, de 200.000 linhas. Você pode acessar os dados em SQL, Python e R.`}
+                  text={`O tamanho da tabela ultrapassou o limite permitido para download, de 1 GB. Você pode acessar os dados em SQL, Python, R e Stata.`}
                 />
               }
 
               <Box
                 as="a"
-                href={downloadUrl}
-                onClick={() => { triggerGAEvent("download_da_tabela",`
-                  {
+                target="_blank"
+                onClick={() => {
+                  if(downloadWarning !== "free" && isUserPro() === false) return plansModal.onOpen()
+                  window.open(`/api/tables/downloadTable?p=${btoa(gcpDatasetID)}&q=${btoa(gcpTableId)}&d=${btoa(downloadPermitted)}&s=${btoa(downloadWarning)}`, "_blank")
+                  triggerGAEvent("download_da_tabela",`{
                     gcp: ${gcpProjectID+"."+gcpDatasetID+"."+gcpTableId},
                     tamanho: ${formatBytes(resource.uncompressedFileSize) || ""},
                     dataset: ${resource?.dataset?._id},
                     table: ${resource?._id},
-                  }`
-                ) }}
-                target="_blank"
+                  }`)
+                }}
                 display="flex"
                 alignItems="center"
                 height="40px"
                 width="fit-content"
                 borderRadius="8px"
-                backgroundColor={downloadNotAllowed ? "#2B8C4D" : "#ACAEB1"}
+                backgroundColor={downloadWarning !== "biggest1gb" ? "#2B8C4D" : "#ACAEB1"}
                 padding="8px 16px"
-                cursor={downloadNotAllowed ? "pointer" : "default"}
+                cursor={downloadWarning !== "biggest1gb" ? "pointer" : "default"}
                 color="#FFF"
                 fill="#FFF"
                 fontFamily="Roboto"
                 fontWeight="500"
                 fontSize="14px"
                 gap="8px"
                 lineHeight="20px"
-                pointerEvents={downloadNotAllowed ? "default" : "none"}
+                pointerEvents={downloadWarning !== "biggest1gb" ? "default" : "none"}
                 _hover={{
                   backgroundColor: "#22703E"
                 }}
@@ -571,7 +633,7 @@ export default function DataInformationQuery({ resource }) {
                   width="24px"
                   height="24px"
                 />
-                  Download da tabela {downloadNotAllowed && `(${formatBytes(resource.uncompressedFileSize)})`}
+                  Download da tabela {downloadWarning !== "biggest1gb" && `(${formatBytes(resource.uncompressedFileSize)})`}
               </Box>
             </Stack>
           </VStack>