forked from jbolda/gatsby-source-airtable
-
Notifications
You must be signed in to change notification settings - Fork 0
/
gatsby-node.js
380 lines (348 loc) · 12.5 KB
/
gatsby-node.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
const Airtable = require("airtable");
const crypto = require("crypto");
const { createRemoteFileNode } = require("gatsby-source-filesystem");
const { map } = require("bluebird");
const mime = require("mime/lite");
exports.sourceNodes = async (
{ actions, createNodeId, store, cache },
{ apiKey, tables, concurrency },
) => {
// tables contain baseId, tableName, tableView, queryName, mapping, tableLinks
const { createNode, setPluginStatus } = actions;
try {
// hoist api so we can use in scope outside of this block
if (!apiKey && process.env.GATSBY_AIRTABLE_API_KEY) {
console.warn(
"\nImplicit setting of GATSBY_AIRTABLE_API_KEY as apiKey will be deprecated in future release, apiKey should be set in gatsby-config.js, please see Readme!",
);
}
var api = await new Airtable({
apiKey: process.env.GATSBY_AIRTABLE_API_KEY || apiKey,
});
} catch (e) {
// airtable uses `assert` which doesn't exit the process,
// but rather just makes gatsby hang. Warn, don't create any
// nodes, but let gatsby continue working
console.warn("\nAPI key is required to connect to Airtable");
return;
}
// exit if tables is not defined
if (tables === undefined || tables.length === 0) {
console.warn(
"\ntables is not defined for gatsby-source-airtable in gatsby-config.js",
);
return;
}
if (concurrency === undefined) {
// Airtable hasn't documented what the rate limit against their attachment servers is.
// They do document that API calls are limited to 5 requests/sec, so the default limit of 5 concurrent
// requests for remote files has been selected in that spirit. A higher value can be set as a plugin
// option in gatsby-config.js
concurrency = 5;
}
console.time(`\nfetch all Airtable rows from ${tables.length} tables`);
let queue = [];
tables.forEach((tableOptions) => {
let base = api.base(tableOptions.baseId);
let table = base(tableOptions.tableName);
let view = tableOptions.tableView || "";
let query = table.select({
view: view,
});
// confirm that the user is using the clean keys
// if they are not, warn them and change it for them
// we can settle the API on clean keys and not have a breaking
// change until the next major version when we remove this
const cleanMapping = !tableOptions.mapping
? null
: Object.keys(tableOptions.mapping).reduce((cleaned, key) => {
let useKey = cleanKey(key);
if (useKey !== key)
console.warn(`
Field names within graphql cannot have spaces. We do not want you to change your column names
within Airtable, but in "Gatsby-land" you will need to always use the "cleaned" key.
On the ${tableOptions.tableName} base ${tableOptions.baseId} 'mapping', we modified the supplied key of
${key} to instead be ${useKey}. Please use ${useKey} in all of your queries. Also, update your config
to use ${useKey} to make this warning go away. See https://github.com/jbolda/gatsby-source-airtable#column-names
for more information.
`);
cleaned[useKey] = tableOptions.mapping[key];
return cleaned;
}, {});
const cleanLinks = !tableOptions.tableLinks
? null
: tableOptions.tableLinks.map((key) => {
let useKey = cleanKey(key);
if (useKey !== key)
console.warn(`
Field names within graphql cannot have spaces. We do not want you to change your column names
within Airtable, but in "Gatsby-land" you will need to always use the "cleaned" key.
On the ${tableOptions.tableName} base ${tableOptions.baseId} 'tableLinks', we modified the supplied key of
${key} to instead be ${useKey}. Please use ${useKey} in all of your queries. Also, update your config
to use ${useKey} to make this warning go away. See https://github.com/jbolda/gatsby-source-airtable#column-names
for more information.
`);
return useKey;
});
// query.all() returns a promise, pass an array for each table with
// both our promise and the queryName and then map reduce at the
// final promise resolution to get queryName onto each row
queue.push(
Promise.all([
query.all(),
tableOptions.queryName,
tableOptions.defaultValues || {},
typeof tableOptions.separateNodeType !== "undefined"
? tableOptions.separateNodeType
: false,
typeof tableOptions.separateMapType !== "undefined"
? tableOptions.separateMapType
: false,
cleanMapping,
cleanLinks,
]),
);
});
// queue has array of promises and when resolved becomes nested arrays
// we flatten the array to return all rows from all tables after mapping
// the queryName to each row
const allRows = await Promise.all(queue)
.then((all) => {
return all.reduce((accumulator, currentValue) => {
return accumulator.concat(
currentValue[0].map((row, rowIndex) => {
row.rowIndex = rowIndex;
row.queryName = currentValue[1]; // queryName from tableOptions above
row.defaultValues = currentValue[2]; // mapping from tableOptions above
row.separateNodeType = currentValue[3]; // separateMapType from tableOptions above
row.separateMapType = currentValue[4]; // create separate node type from tableOptions above
row.mapping = currentValue[5]; // mapping from tableOptions above
row.tableLinks = currentValue[6]; // tableLinks from tableOptions above
return row;
}),
);
}, []);
})
.catch((e) => {
throw e;
return;
});
console.timeEnd(`\nfetch all Airtable rows from ${tables.length} tables`);
setPluginStatus({
status: {
lastFetched: new Date().toJSON(),
},
});
// Use the map function for arrays of promises imported from Bluebird.
// Using the concurrency option protects against being blocked from Airtable's
// file attachment servers for large numbers of requests.
return map(
allRows,
async (row) => {
// don't love mutating the row here, but
// not ready to refactor yet to clean this up
// (happy to take a PR!)
row.fields = {
...row.defaultValues,
...row.fields,
};
let processedData = await processData(row, {
createNodeId,
createNode,
store,
cache,
});
if (row.separateNodeType && (!row.queryName || row.queryName === "")) {
console.warn(
`You have opted into separate node types, but not specified a queryName.
We use the queryName to suffix to node type. Without a queryName, it will act like separateNodeType is false.`,
);
}
const node = {
id: createNodeId(`Airtable_${row.id}`),
parent: null,
table: row._table.name,
recordId: row.id,
queryName: row.queryName,
rowIndex: row.rowIndex,
children: [],
internal: {
type: `Airtable${
row.separateNodeType ? cleanType(row.queryName) : ""
}`,
contentDigest: crypto
.createHash("md5")
.update(JSON.stringify(row))
.digest("hex"),
},
data: processedData.data,
};
createNode(node);
await Promise.all(processedData.childNodes).then((nodes) => {
nodes.forEach((node) => createNode(node));
});
},
{ concurrency: concurrency },
);
};
const processData = async (row, { createNodeId, createNode, store, cache }) => {
let data = row.fields;
let tableLinks = row.tableLinks;
let fieldKeys = Object.keys(data);
let processedData = {};
let childNodes = [];
fieldKeys.forEach((key) => {
// once in "Gatsby-land" we want to use the cleanKey
// consistently everywhere including in configs
// this key that we clean comes from Airtable
// at this point, all user option keys should be clean
const cleanedKey = cleanKey(key);
let useKey;
// deals with airtable linked fields,
// these will be airtable IDs
if (tableLinks && tableLinks.includes(cleanedKey)) {
useKey = `${cleanedKey}___NODE`;
// `data` is direct from Airtable so we don't use
// the cleanKey here
processedData[useKey] = data[key].map((id) =>
createNodeId(`Airtable_${id}`),
);
} else if (row.mapping && row.mapping[cleanedKey]) {
// A child node comes from the mapping, where we want to
// define a separate node in gatsby that is available
// for transforming. This will let other plugins pick
// up on that node to add nodes.
// row contains `data` which is direct from Airtable
// so we pass down the raw instead of the cleanKey here
let checkedChildNode = checkChildNode(key, row, processedData, {
createNodeId,
createNode,
store,
cache,
});
childNodes.push(checkedChildNode);
} else {
// `data` is direct from Airtable so we don't use
// the cleanKey here
processedData[cleanedKey] = data[key];
}
});
// wait for all of the children to finish
await Promise.all(childNodes);
// where childNodes returns an array of objects
return { data: processedData, childNodes: childNodes };
};
const checkChildNode = async (
key,
row,
processedData,
{ createNodeId, createNode, store, cache },
) => {
let data = row.fields;
let mapping = row.mapping;
let cleanedKey = cleanKey(key);
let localFiles = await localFileCheck(key, row, {
createNodeId,
createNode,
store,
cache,
});
processedData[`${cleanedKey}___NODE`] = createNodeId(
`AirtableField_${row.id}_${cleanedKey}`,
);
return buildNode(
localFiles,
row,
cleanedKey,
data[key],
mapping[key],
createNodeId,
);
};
const localFileCheck = async (
key,
row,
{ createNodeId, createNode, store, cache },
) => {
let data = row.fields;
let mapping = row.mapping;
let cleanedKey = cleanKey(key);
if (mapping[cleanedKey] === "fileNode") {
try {
let fileNodes = [];
// where data[key] is the array of attachments
// `data` is direct from Airtable so we don't use
// the cleanKey here
data[key].forEach((attachment) => {
const ext = mime.getExtension(attachment.type); // unknown type returns null
let attachmentNode = createRemoteFileNode({
url: attachment.url,
store,
cache,
createNode,
createNodeId,
ext: !!ext ? `.${ext}` : undefined,
});
fileNodes.push(attachmentNode);
});
// Adds a field `localFile` to the node
// ___NODE tells Gatsby that this field will link to another nodes
const resolvedFileNodes = await Promise.all(fileNodes);
const localFiles = resolvedFileNodes.map(
(attachmentNode) => attachmentNode.id,
);
return localFiles;
} catch (e) {
console.log(
"You specified a fileNode, but we caught an error. First check that you have gatsby-source-filesystem installed?\n",
e,
);
}
}
return;
};
const buildNode = (localFiles, row, cleanedKey, raw, mapping, createNodeId) => {
const nodeType = row.separateNodeType
? `Airtable${cleanKey(row.queryName ? row.queryName : row._table.name)}`
: "Airtable";
if (localFiles) {
return {
id: createNodeId(`AirtableField_${row.id}_${cleanedKey}`),
parent: createNodeId(`Airtable_${row.id}`),
children: [],
raw: raw,
localFiles___NODE: localFiles,
internal: {
type: `AirtableField${row.separateMapType ? cleanType(mapping) : ""}`,
mediaType: mapping,
content: typeof raw === "string" ? raw : JSON.stringify(raw),
contentDigest: crypto
.createHash("md5")
.update(JSON.stringify(row))
.digest("hex"),
},
};
} else {
return {
id: createNodeId(`AirtableField_${row.id}_${cleanedKey}`),
parent: createNodeId(`Airtable_${row.id}`),
children: [],
raw: raw,
internal: {
type: `AirtableField${row.separateMapType ? cleanType(mapping) : ""}`,
mediaType: mapping,
content: typeof raw === "string" ? raw : JSON.stringify(raw),
contentDigest: crypto
.createHash("md5")
.update(JSON.stringify(row))
.digest("hex"),
},
};
}
};
const cleanKey = (key, data) => {
return key.replace(/ /g, "_");
};
const cleanType = (key) => {
return !key ? "" : key.replace(/[ /+]/g, "");
};