Merge pull request #227 from IQSS/feature/224-integration-files-tabul…

…ar-data

224 - Integration files tabular data

src/files/domain/models/File.ts

@@ -108,7 +108,7 @@ export class FileEmbargo {
 export interface FileTabularData {
   variablesCount: number
   observationsCount: number
-  unf: string
+  unf?: string
 }
 
 export enum FileLabelType {
@@ -154,12 +154,12 @@ export class File {
     readonly type: FileType,
     readonly size: FileSize,
     readonly date: FileDate,
-    public downloadCount: number,
+    readonly downloadCount: number,
     readonly labels: FileLabel[],
     public readonly isDeleted: boolean,
     public readonly ingest: FileIngest,
     readonly checksum?: FileChecksum,
-    public thumbnail?: string,
+    readonly thumbnail?: string,
     readonly directory?: string,
     readonly embargo?: FileEmbargo,
     readonly tabularData?: FileTabularData,

src/files/infrastructure/FileJSDataverseRepository.ts

@@ -1,5 +1,5 @@
 import { FileRepository } from '../domain/repositories/FileRepository'
-import { File, FilePublishingStatus } from '../domain/models/File'
+import { File } from '../domain/models/File'
 import { FilesCountInfo } from '../domain/models/FilesCountInfo'
 import { FilePaginationInfo } from '../domain/models/FilePaginationInfo'
 import { FileUserPermissions } from '../domain/models/FileUserPermissions'
@@ -10,7 +10,10 @@ import {
   getDatasetFilesTotalDownloadSize,
   getFileDownloadCount,
   getFileUserPermissions,
-  ReadError
+  ReadError,
+  File as JSFile,
+  getFileDataTables,
+  FileDataTable as JSFileTabularData
 } from '@iqss/dataverse-client-javascript'
 import { FileCriteria } from '../domain/models/FileCriteria'
 import { DomainFileMapper } from './mappers/DomainFileMapper'
@@ -30,7 +33,6 @@ export class FileJSDataverseRepository implements FileRepository {
     criteria: FileCriteria = new FileCriteria()
   ): Promise<File[]> {
     const jsPagination = DomainFileMapper.toJSPagination(paginationInfo)
-
     return getDatasetFiles
       .execute(
         datasetPersistentId,
@@ -41,48 +43,54 @@ export class FileJSDataverseRepository implements FileRepository {
         DomainFileMapper.toJSFileSearchCriteria(criteria),
         DomainFileMapper.toJSFileOrderCriteria(criteria.sortBy)
       )
-      .then((jsFiles) => jsFiles.map((jsFile) => JSFileMapper.toFile(jsFile, datasetVersion)))
-      .then((files) => FileJSDataverseRepository.getAllWithDownloadCount(files))
-      .then((files) => FileJSDataverseRepository.getAllWithThumbnail(files))
+      .then((jsFiles) =>
+        Promise.all([
+          jsFiles,
+          FileJSDataverseRepository.getAllDownloadCount(jsFiles),
+          FileJSDataverseRepository.getAllThumbnails(jsFiles),
+          FileJSDataverseRepository.getAllTabularData(jsFiles)
+        ])
+      )
+      .then(([jsFiles, downloadCounts, thumbnails, jsTabularData]) =>
+        jsFiles.map((jsFile, index) =>
+          JSFileMapper.toFile(
+            jsFile,
+            datasetVersion,
+            downloadCounts[index],
+            thumbnails[index],
+            jsTabularData[index]
+          )
+        )
+      )
       .catch((error: ReadError) => {
         throw new Error(error.message)
       })
   }
 
-  private static getAllWithDownloadCount(files: File[]): Promise<File[]> {
+  private static getAllTabularData(
+    jsFiles: JSFile[]
+  ): Promise<(JSFileTabularData[] | undefined)[]> {
     return Promise.all(
-      files.map((file) =>
-        FileJSDataverseRepository.getDownloadCountById(file.id, file.version.publishingStatus).then(
-          (downloadCount) => {
-            file.downloadCount = downloadCount
-            return file
-          }
-        )
+      jsFiles.map((jsFile) =>
+        jsFile.tabularData ? getFileDataTables.execute(jsFile.id) : undefined
       )
     )
   }
 
-  private static getDownloadCountById(
-    id: number,
-    publishingStatus: FilePublishingStatus
-  ): Promise<number> {
-    if (publishingStatus === FilePublishingStatus.RELEASED) {
-      return getFileDownloadCount.execute(id).then((downloadCount) => Number(downloadCount))
-    }
-    return Promise.resolve(0)
-  }
-
-  private static getAllWithThumbnail(files: File[]): Promise<File[]> {
+  private static getAllDownloadCount(jsFiles: JSFile[]): Promise<number[]> {
     return Promise.all(
-      files.map((file) =>
-        FileJSDataverseRepository.getThumbnailById(file.id).then((thumbnail) => {
-          file.thumbnail = thumbnail
-          return file
-        })
+      jsFiles.map((jsFile) =>
+        jsFile.publicationDate
+          ? getFileDownloadCount.execute(jsFile.id).then((downloadCount) => Number(downloadCount))
+          : 0
       )
     )
   }
 
+  private static getAllThumbnails(jsFiles: JSFile[]): Promise<(string | undefined)[]> {
+    return Promise.all(jsFiles.map((jsFile) => this.getThumbnailById(jsFile.id)))
+  }
+
   private static getThumbnailById(id: number): Promise<string | undefined> {
     return fetch(`${this.DATAVERSE_BACKEND_URL}/api/access/datafile/${id}?imageThumb=400`)
       .then((response) => {

src/files/infrastructure/mappers/JSFileMapper.ts

@@ -11,6 +11,7 @@ import {
   FilePublishingStatus,
   FileSize,
   FileSizeUnit,
+  FileTabularData,
   FileType,
   FileVersion
 } from '../../domain/models/File'
@@ -22,7 +23,8 @@ import {
   FileContentTypeCount as JSFileContentTypeCount,
   FileCategoryNameCount as JSFileCategoryNameCount,
   FileAccessStatusCount as JSFileAccessStatusCount,
-  FileAccessStatus as JSFileAccessStatus
+  FileAccessStatus as JSFileAccessStatus,
+  FileDataTable as JSFileTabularData
 } from '@iqss/dataverse-client-javascript'
 import { DatasetPublishingStatus, DatasetVersion } from '../../../dataset/domain/models/Dataset'
 import { FileUserPermissions } from '../../domain/models/FileUserPermissions'
@@ -35,7 +37,13 @@ import {
 import { FileAccessOption, FileTag } from '../../domain/models/FileCriteria'
 
 export class JSFileMapper {
-  static toFile(jsFile: JSFile, datasetVersion: DatasetVersion): File {
+  static toFile(
+    jsFile: JSFile,
+    datasetVersion: DatasetVersion,
+    downloadsCount: number,
+    thumbnail?: string,
+    jsTabularData?: JSFileTabularData[]
+  ): File {
     return new File(
       this.toFileId(jsFile.id),
       this.toFileVersion(jsFile.version, datasetVersion, jsFile.publicationDate),
@@ -44,15 +52,15 @@ export class JSFileMapper {
       this.toFileType(jsFile.contentType, jsFile.originalFormatLabel),
       this.toFileSize(jsFile.sizeBytes),
       this.toFileDate(jsFile.creationDate, jsFile.publicationDate, jsFile.embargo),
-      this.toFileDownloads(),
+      this.toFileDownloads(downloadsCount),
       this.toFileLabels(jsFile.categories, jsFile.tabularTags),
-      false, // TODO - Implement this when it is added to js-dataverse
+      this.toFileIsDeleted(jsFile.deleted),
       { status: FileIngestStatus.NONE }, // TODO - Implement this when it is added to js-dataverse
       this.toFileChecksum(jsFile.checksum),
-      this.toFileThumbnail(),
+      this.toFileThumbnail(thumbnail),
       this.toFileDirectory(jsFile.directoryLabel),
       this.toFileEmbargo(jsFile.embargo),
-      this.toFileTabularData(),
+      this.toFileTabularData(jsTabularData),
       this.toFileDescription(jsFile.description)
     )
   }
@@ -132,8 +140,8 @@ export class JSFileMapper {
     throw new Error('File date not found')
   }
 
-  static toFileDownloads(): number {
-    return 0 // This is always 0 because the downloads come from a different endpoint
+  static toFileDownloads(downloadsCount: number): number {
+    return downloadsCount
   }
 
   static toFileLabels(jsFileCategories?: string[], jsFileTabularTags?: string[]): FileLabel[] {
@@ -159,8 +167,8 @@ export class JSFileMapper {
     return undefined
   }
 
-  static toFileThumbnail(): undefined {
-    return undefined // This is always undefined because the thumbnails come from a different endpoint
+  static toFileThumbnail(thumbnail?: string): string | undefined {
+    return thumbnail
   }
 
   static toFileDirectory(jsFileDirectory: string | undefined): string | undefined {
@@ -174,8 +182,15 @@ export class JSFileMapper {
     return undefined
   }
 
-  static toFileTabularData(): undefined {
-    return undefined // This is always undefined because the tabular data comes from a different endpoint
+  static toFileTabularData(jsTabularData?: JSFileTabularData[]): FileTabularData | undefined {
+    if (jsTabularData === undefined) {
+      return undefined
+    }
+    return {
+      variablesCount: jsTabularData[0].varQuantity ?? 0,
+      observationsCount: jsTabularData[0].caseQuantity ?? 0,
+      unf: jsTabularData[0].UNF
+    }
   }
 
   static toFileDescription(jsFileDescription?: string): string | undefined {
@@ -230,4 +245,8 @@ export class JSFileMapper {
         return FileAccessOption.EMBARGOED_RESTRICTED
     }
   }
+
+  static toFileIsDeleted(jsFileIsDeleted: boolean | undefined): boolean {
+    return jsFileIsDeleted ?? false
+  }
 }

src/sections/dataset/dataset-files/files-table/file-info/file-info-cell/file-info-data/FileTabularData.tsx

@@ -15,7 +15,7 @@ export function FileTabularData({
     <div>
       {tabularData.variablesCount} {t('table.tabularData.variables')},{' '}
       {tabularData.observationsCount} {t('table.tabularData.observations')}{' '}
-      <CopyToClipboardButton text={tabularData.unf} />
+      {tabularData.unf && <CopyToClipboardButton text={tabularData.unf} />}
     </div>
   )
 }

src/stories/dataset/dataset-citation/DatasetCitation.stories.tsx

@@ -34,7 +34,6 @@ export const Default: Story = {
 export const WithThumbnail: Story = {
   render: () => {
     const dataset = DatasetMother.createRealistic({ thumbnail: faker.image.imageUrl() })
-    console.log(dataset)
     return (
       <div>
         <br></br>

tests/e2e-integration/integration/files/FileJSDataverseRepository.spec.ts

@@ -100,6 +100,7 @@ describe('File JSDataverse Repository', () => {
             expect(file.embargo).to.deep.equal(expectedFile.embargo)
             expect(file.tabularData).to.deep.equal(expectedFile.tabularData)
             expect(file.description).to.deep.equal(expectedFile.description)
+            expect(file.isDeleted).to.deep.equal(expectedFile.isDeleted)
           })
         })
     })
@@ -260,7 +261,6 @@ describe('File JSDataverse Repository', () => {
       await fileRepository
         .getAllByDatasetPersistentId(dataset.persistentId, dataset.version)
         .then((files) => {
-          console.log(files)
           expect(files[0].thumbnail).to.not.be.undefined
         })
     })
@@ -288,8 +288,7 @@ describe('File JSDataverse Repository', () => {
         })
     })
 
-    it.skip('gets all the files by dataset persistentId when files are tabular data', async () => {
-      // TODO - Implement this when isTabularData flag is added to js-dataverse response
+    it('gets all the files by dataset persistentId when files are tabular data', async () => {
       const datasetResponse = await DatasetHelper.createWithFiles(FileHelper.createMany(1, 'csv'))
       if (!datasetResponse.files) throw new Error('Files not found')
 
@@ -300,12 +299,17 @@ describe('File JSDataverse Repository', () => {
         .getAllByDatasetPersistentId(dataset.persistentId, dataset.version)
         .then((files) => {
           const expectedTabularData = {
-            variablesCount: 1,
-            observationsCount: 0,
-            unf: 'some'
+            variablesCount: 7,
+            observationsCount: 10
           }
           files.forEach((file) => {
-            expect(file.tabularData).to.deep.equal(expectedTabularData)
+            expect(file.tabularData?.variablesCount).to.deep.equal(
+              expectedTabularData.variablesCount
+            )
+            expect(file.tabularData?.observationsCount).to.deep.equal(
+              expectedTabularData.observationsCount
+            )
+            expect(file.tabularData?.unf).to.not.be.undefined
           })
         })
     })
@@ -433,6 +437,29 @@ describe('File JSDataverse Repository', () => {
           expect(files.length).to.equal(1)
         })
     })
+
+    it.skip('gets all the files when they are deleted', async () => {
+      // This test is failing because js-dataverse deleted property always returns undefined
+      // TODO: Remove the skip once the issue is fixed
+      const datasetResponse = await DatasetHelper.createWithFiles(FileHelper.createMany(1))
+
+      await DatasetHelper.publish(datasetResponse.persistentId)
+      await TestsUtils.wait(2000) // Wait for the dataset to be published
+
+      const dataset = await datasetRepository.getByPersistentId(datasetResponse.persistentId)
+      if (!dataset) throw new Error('Dataset not found')
+
+      if (!datasetResponse.files) throw new Error('Files not found')
+      datasetResponse.files.map((file) => FileHelper.delete(file.id))
+
+      await fileRepository
+        .getAllByDatasetPersistentId(dataset.persistentId, dataset.version)
+        .then((files) => {
+          files.forEach((file) => {
+            expect(file.isDeleted).to.equal(true)
+          })
+        })
+    })
   })
 
   describe('Get file user permissions by id', () => {

tests/e2e-integration/shared/files/FileHelper.ts

@@ -152,4 +152,8 @@ export class FileHelper extends DataverseApiHelper {
       'multipart/form-data'
     )
   }
+
+  static async delete(id: number) {
+    return this.request<FileResponse>(`/files/${id}`, 'DELETE')
+  }
 }