feat(taxonomy): transform to import data [NONE]

.gitignore

@@ -192,3 +192,4 @@ contentful-import-error-log-*.json
 
 .idea
 reports
+data

docs/organization/transform/README.md

@@ -0,0 +1,9 @@
+# Contentful CLI - `organization transform` command
+
+Transform data to organization entities. This is helpful if you want to transform any data to valid import data.
+
+## Example
+
+```sh
+contentful organization transform
+```

docs/organization/transform/examples/add-single-concept.js

@@ -0,0 +1,5 @@
+module.exports = function (context) {
+  context.addConcept({
+    prefLabel: { 'en-US': 'Hello world' + Date.now().toString() }
+  })
+}

docs/organization/transform/examples/example-spreadsheet.csv

@@ -0,0 +1,10 @@
+Animal Group,Sub-Category,Animal Name,Skills,Knowledge,Opportunities
+Mammals,Carnivores,Beagle,"Handling large carnivores, Feeding protocols, Safety procedures","Diet specifics, Social structure, Natural habitat","Conservation programs, Public education, Specialized training"
+Mammals,Herbivores,Elephant,"Large animal handling, Medical care for large mammals","Dietary needs, Social dynamics, Habitat requirements","Conservation work, Veterinary specialization, Enrichment program development"
+Mammals,Herbivores,Giraffe,"Giraffe care techniques, Enclosure management","Feeding behavior, Social interaction, Habitat","Conservation initiatives, Research projects, Educational outreach"
+Birds,Aquatic,Penguin,"Cold environment handling, Feeding techniques","Breeding habits, Climate adaptation, Diet","Antarctic research, Breeding programs, Public engagement"
+Reptiles,Venomous,Snake,"Safe handling, Venom management","Species identification, Venom properties, Behavior","Venom extraction, Educational demonstrations, Conservation breeding"
+Reptiles,Non-Venomous,Tortoise,"Shell care, Enclosure maintenance","Longevity, Diet, Behavior","Conservation work, Breeding programs, Public education"
+Fish,Freshwater,Koi,"Water quality management, Feeding","Breeding, Habitat needs, Behavior","Aquatic habitat research, Public education, Koi shows"
+Amphibians,Frogs,Poison Dart Frog,"Safe handling, Enclosure management","Toxicity levels, Breeding, Habitat","Conservation efforts, Public awareness, Research projects"
+Invertebrates,Arthropods,Tarantula,"Safe handling, Enclosure care","Venom knowledge, Behavior, Diet","Educational programs, Breeding, Conservation"

docs/organization/transform/examples/load-csv-file.js

@@ -0,0 +1,19 @@
+module.exports = async function ({ csv, concepts, updateConcept, addConcept }) {
+  const { data } = await csv.parseFile('./example-spreadsheet.csv')
+
+  for (const row of data) {
+    const existingConcept = concepts.find(
+      c => c.prefLabel['en-US'] === row['Animal Name']
+    )
+    if (existingConcept) {
+      updateConcept({
+        ...existingConcept,
+        prefLabel: { 'en-US': row['Animal Name'] }
+      })
+    } else {
+      addConcept({
+        prefLabel: { 'en-US': row['Animal Name'] }
+      })
+    }
+  }
+}

lib/cmds/organization_cmds/transform.ts

@@ -0,0 +1,214 @@
+import { ConceptProps, ConceptSchemeProps } from 'contentful-management'
+import { Listr } from 'listr2'
+import { noop } from 'lodash'
+import * as fs from 'node:fs'
+import path, { relative, resolve } from 'path'
+import type { Argv } from 'yargs'
+import { handleAsyncError as handle } from '../../utils/async'
+import { createPlainClient } from '../../utils/contentful-clients'
+import { copyright } from '../../utils/copyright'
+import { cursorPaginate } from '../../utils/cursor-pagninate'
+import { ensureDir, getPath, writeFileP } from '../../utils/fs'
+import { getHeadersFromOption } from '../../utils/headers'
+import { success } from '../../utils/log'
+import Papa, { ParseConfig } from 'papaparse'
+
+module.exports.command = 'transform'
+
+module.exports.desc = 'transform data to organization entities'
+
+module.exports.builder = (yargs: Argv) => {
+  return yargs
+    .usage('Usage: contentful organization export')
+    .option('management-token', {
+      alias: 'mt',
+      describe: 'Contentful management API token',
+      type: 'string'
+    })
+    .option('organization-id', {
+      alias: 'oid',
+      describe: 'ID of Organization with source data',
+      type: 'string',
+      demandOption: true
+    })
+    .option('header', {
+      alias: 'H',
+      type: 'string',
+      describe: 'Pass an additional HTTP Header'
+    })
+    .option('transform-file', {
+      alias: 't',
+      type: 'string',
+      describe: 'Transform file'
+    })
+    .option('output-file', {
+      alias: 'o',
+      type: 'string',
+      describe:
+        'Output file. It defaults to ./data/<timestamp>-<organization-id>.json'
+    })
+    .epilog(
+      [
+        'See more at:',
+        'https://github.com/contentful/contentful-cli/tree/master/docs/organization/export',
+        copyright
+      ].join('\n')
+    )
+}
+
+const createFsContext = (transformPath: string) => ({
+  readFile: (path: string) => {
+    return fs.promises.readFile(resolve(transformPath, path), 'utf-8')
+  }
+})
+
+const createCSVContext = (transformPath: string) => ({
+  parse: (csvString: string, config?: ParseConfig) => {
+    return Papa.parse(csvString, config)
+  },
+  parseFile: async (filePath: string, config?: ParseConfig) => {
+    const content = await fs.promises.readFile(
+      resolve(transformPath, filePath),
+      'utf-8'
+    )
+    return Papa.parse(content, config)
+  }
+})
+
+interface TransformContext {
+  concepts: ConceptProps[]
+  conceptSchemes: ConceptSchemeProps[]
+  updateConcept: (concept: ConceptProps) => void
+  addConcept: (concept: Omit<ConceptProps, 'sys'>) => void
+  deleteConcept: (conceptId: string) => void
+  fs: ReturnType<typeof createFsContext>
+  csv: ReturnType<typeof createCSVContext>
+}
+
+type TransformFunction = (transformContext: TransformContext) => Promise<void>
+
+interface Params {
+  context: { managementToken: string }
+  header: string
+  organizationId: string
+  outputFile: string
+  transformFile: string
+}
+
+async function organizationExport({
+  context,
+  header,
+  organizationId,
+  outputFile,
+  transformFile
+}: Params) {
+  const { managementToken } = context
+
+  const client = await createPlainClient({
+    accessToken: managementToken,
+    feature: 'organization-export',
+    headers: getHeadersFromOption(header),
+    throttle: 8,
+    logHandler: noop
+  })
+
+  const outputTarget = getPath(
+    outputFile || path.join('data', `${Date.now()}-${organizationId}.json`)
+  )
+  await ensureDir(path.dirname(outputTarget))
+
+  const transform: { default: TransformFunction } = await import(
+    resolve(__dirname, relative(__dirname, transformFile))
+  )
+
+  const tasks = new Listr(
+    [
+      {
+        title: 'Loading Organization',
+        task: async (_, rootTask) => {
+          return new Listr([
+            {
+              title: 'Fetching Concepts',
+              task: async (ctx, task) => {
+                ctx.concepts = await cursorPaginate({
+                  queryPage: pageUrl =>
+                    client.concept.getMany({
+                      organizationId,
+                      query: { pageUrl }
+                    })
+                })
+                task.title = `${ctx.concepts.length} Concepts fetched`
+              }
+            },
+            {
+              title: 'Fetching Concept Schemes',
+              task: async (ctx, task) => {
+                ctx.conceptSchemes = await cursorPaginate({
+                  queryPage: pageUrl =>
+                    client.conceptScheme.getMany({
+                      organizationId,
+                      query: { pageUrl }
+                    })
+                })
+                task.title = `${ctx.conceptSchemes.length} Concept Schemes fetched`
+              }
+            },
+            {
+              title: 'Execute transform',
+              task: async (ctx, task) => {
+                const { add, update, del } = ctx.result.concepts
+                await transform.default({
+                  fs: createFsContext(
+                    path.dirname(
+                      resolve(__dirname, relative(__dirname, transformFile))
+                    )
+                  ),
+                  csv: createCSVContext(
+                    path.dirname(
+                      resolve(__dirname, relative(__dirname, transformFile))
+                    )
+                  ),
+                  concepts: ctx.concepts,
+                  conceptSchemes: ctx.conceptSchemes,
+                  updateConcept: async concept => update.push(concept),
+                  addConcept: async concept => add.push(concept),
+                  deleteConcept: async conceptId => del.push(conceptId)
+                })
+                task.title = `Transform executed on ${
+                  ctx.result.concepts.add.length +
+                  ctx.result.concepts.update.length +
+                  ctx.result.concepts.del.length
+                } Concepts`
+                rootTask.title = 'Organization data transformed'
+              }
+            }
+          ])
+        }
+      }
+    ],
+    { rendererOptions: { showTimer: true, collapse: false } }
+  )
+
+  const ctx = await tasks.run({
+    concepts: [],
+    conceptSchemes: [],
+    result: { concepts: { add: [], update: [], del: [] } }
+  })
+  await writeFileP(
+    outputTarget,
+    JSON.stringify(
+      {
+        concepts: [...ctx.result.concepts.add, ...ctx.result.concepts.update],
+        conceptSchemes: ctx.conceptSchemes
+      },
+      null,
+      2
+    )
+  )
+  console.log('\n')
+  success(`✅ Organization data exported to ${outputTarget}`)
+}
+
+module.exports.organizationExport = organizationExport
+
+module.exports.handler = handle(organizationExport)

lib/config.js

@@ -17,6 +17,7 @@ module.exports = {
     'feedback',
     'organization list',
     'organization export',
+    'organization transform',
     'space create',
     'space list',
     'space use'

package.json

@@ -50,6 +50,7 @@
     "@types/listr": "^0.14.4",
     "@types/lodash": "^4.14.191",
     "@types/node": "^22.4.1",
+    "@types/papaparse": "^5.3.14",
     "@types/yargs": "^13.0.12",
     "@typescript-eslint/eslint-plugin": "^5.31.0",
     "@typescript-eslint/parser": "^5.31.0",
@@ -108,6 +109,7 @@
     "marked": "^7.0.0",
     "mkdirp": "^3.0.0",
     "open": "^8.4.2",
+    "papaparse": "^5.4.1",
     "path": "^0.12.7",
     "prettier": "^2.0.2",
     "recast": "^0.23.2",