Skip to content

Commit

Permalink
add academies & uu
Browse files Browse the repository at this point in the history
  • Loading branch information
@jerem1508
jerem1508 committed Oct 2, 2023
1 parent 8bc7ccd commit be251d3
Show file tree
Hide file tree
Showing 9 changed files with 121 additions and 201,524 deletions.
62,152 changes: 0 additions & 62,152 deletions scripts/countries.geo.json
View file

This file was deleted.

110,526 changes: 0 additions & 110,526 deletions scripts/departments.geo.json
View file

This file was deleted.

4 changes: 2 additions & 2 deletions scripts/import-geographical-categories-countries.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ async function getPaysageIds(existingIdsCount) {
}

async function getCountriesToUpgrade() {
return db.collection(MONGO_TARGET_COLLECTION_NAME).find({ level: 'pays' }).toArray();
return db.collection(MONGO_TARGET_COLLECTION_NAME).find({ level: 'country' }).toArray();
}

async function treatment() {
Expand All @@ -43,7 +43,7 @@ async function treatment() {
db.collection(MONGO_TARGET_COLLECTION_NAME).updateOne({ originalId: geo.originalId }, { $set: geo }, { upsert: true })
));

await Promise.all(promises);
await Promise.all(promises);
}

console.log('--- START ---');
Expand Down
55 changes: 36 additions & 19 deletions ...aphical-categories-regions-departments.js → ...tegories-regions-departments-academies.js
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Example : NODE_ENV=development MONGO_URI="mongodb://localhost:27017" MONGO_DBNAME="paysage" node --experimental-specifier-resolution=node scripts/import-geographical-categories-regions-departments.js
// Example : NODE_ENV=development MONGO_URI="mongodb://localhost:27017" MONGO_DBNAME="paysage" node --experimental-specifier-resolution=node scripts/import-geographical-categories-regions-departments-academies.js
// source regions: https://data.opendatasoft.com/explore/dataset/georef-france-region%40public/export/?disjunctive.reg_name
// Source departments: https://data.opendatasoft.com/explore/dataset/georef-france-departement%40public/table/?disjunctive.reg_name&disjunctive.dep_name&sort=year
import 'dotenv/config';
Expand All @@ -9,8 +9,9 @@ import BaseMongoCatalog from '../src/api/commons/libs/base.mongo.catalog';
const MONGO_SOURCE_COLLECTION_NAME = 'geocodes';
const MONGO_TARGET_COLLECTION_NAME = 'geographicalcategories';

import regions from './regions.geo.json' assert { type: "json" };
import departments from './departments.geo.json' assert { type: "json" };
import regions from './data/regions.geo.json' assert { type: "json" };
import departments from './data/departments.geo.json' assert { type: "json" };
import academies from './data/fr-en-contour-academies-2020.geo.json' assert { type: "json" };

const configs = [
{
Expand All @@ -32,13 +33,16 @@ const configs = [
sourceIdLength: 3,
sourceNameField: 'dep_nom',
},
// {
// prefix: 'C',
// level: 'department',
// sourceIdField: 'uucr_id',
// sourceNameField: 'uucr_nom',
// url: ' ',
// },
{
data: academies,
geoCodeField: 'code_academie',
level: 'academy',
parentIdField: 'reg_id',
prefix: 'A',
sourceIdField: 'aca_id',
sourceIdLength: 3,
sourceNameField: 'aca_nom',
},
];

async function treatment() {
Expand All @@ -47,21 +51,34 @@ async function treatment() {
// Load uniq from Mongo geocodes
const uniqueGeoIds = await db.collection(MONGO_SOURCE_COLLECTION_NAME).distinct(config.sourceIdField);
const uniqueGeos = await Promise.all(
uniqueGeoIds.map((uniqueRegionId) => db.collection(MONGO_SOURCE_COLLECTION_NAME).findOne({ [config.sourceIdField]: uniqueRegionId })),
uniqueGeoIds.map((uniqueId) => db.collection(MONGO_SOURCE_COLLECTION_NAME).findOne({ [config.sourceIdField]: uniqueId })),
);
// Generate as many ids as needed
const catalog = new BaseMongoCatalog({ db, collection: '_catalog' });
const allIds = await Promise.all(
uniqueGeos.map(() => catalog.getUniqueId(MONGO_TARGET_COLLECTION_NAME, 5)),
);
const p = uniqueGeos.map((geo, index) => ({
geometry: data.features.find((geojson) => `${config.prefix}${geojson.properties[config.geoCodeField][0].length < config.sourceIdLength ? '0' : ''}${geojson.properties[config.geoCodeField][0]}` === geo[config.sourceIdField])?.geometry || null,
id: allIds[index],
level: config.level,
nameFr: geo[config.sourceNameField],
originalId: geo[config.sourceIdField],
parentOriginalId: config.level === 'region' ? 'FRA' : geo[config.parentIdField],
})).map((geo) => db.collection(MONGO_TARGET_COLLECTION_NAME).updateOne({ originalId: geo.originalId }, { $set: geo }, { upsert: true }));
const p = uniqueGeos.map((geo, index) => {
if (config.level === 'academy') {
return {
geometry: data.features.find((geojson) => `A${geojson.properties.code_academie}` === geo[config.sourceIdField])?.geometry || null,
id: allIds[index],
level: config.level,
nameFr: geo[config.sourceNameField],
originalId: geo[config.sourceIdField],
parentOriginalId: geo[config.parentIdField],
};
}
return {
geometry: data.features.find((geojson) => `${config.prefix}${geojson.properties[config.geoCodeField][0].length < config.sourceIdLength ? '0' : ''}${geojson.properties[config.geoCodeField][0]}` === geo[config.sourceIdField])?.geometry || null,
id: allIds[index],
level: config.level,
nameFr: geo[config.sourceNameField],
originalId: geo[config.sourceIdField],
parentOriginalId: config.level === 'region' ? 'FRA' : geo[config.parentIdField],
};
}
).map((geo) => db.collection(MONGO_TARGET_COLLECTION_NAME).updateOne({ originalId: geo.originalId }, { $set: geo }, { upsert: true }));
await Promise.all(p);
});
await Promise.all(promises);
Expand Down
80 changes: 80 additions & 0 deletions scripts/import-geographical-categories-uu.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import fs from 'fs';

import 'dotenv/config';
import refUU from './data/referentiel-geographique-francais-communes-unites-urbaines-aires-urbaines-depart.json' assert { type: "json" };

import { client, db } from '../src/services/mongo.service';
import BaseMongoCatalog from '../src/api/commons/libs/base.mongo.catalog';

const MONGO_TARGET_COLLECTION_NAME = 'geographicalcategories';

async function getUUToUpgrade() {
return db.collection(MONGO_TARGET_COLLECTION_NAME).find({ level: 'urbanUnity' }).toArray();
}

async function getPaysageIds(existingIdsCount, listUU) {
const catalog = new BaseMongoCatalog({ db, collection: '_catalog' });
return Promise.all(
Object.keys(listUU).slice(0, Object.keys(listUU).length - existingIdsCount).map(() => catalog.getUniqueId(MONGO_TARGET_COLLECTION_NAME, 5)),
);
}

async function treatment() {
const allUu = {};
refUU.forEach((el) => {
if (el.uu_code) {
if (!allUu[el.uu_code]) {
allUu[el.uu_code] = {
list: [{
code_com: el.com_code,
geometry: el.geom.geometry || null,
}],
uucr_nom: el.uucr_nom
}
} else {
allUu[el.uu_code].list.push({
code_com: el.com_code,
geometry: el.geom.geometry || null,
});
}
}
});

const uUToUpgrade = await getUUToUpgrade();

// Get paysage ids
const ids = await getPaysageIds(uUToUpgrade.length, allUu);
console.log(ids);
const promises = Object.keys(allUu).map((urbanUnity, index) => ({
geometry: { coordinates: allUu[urbanUnity].list.map((commune) => commune.geometry.coordinates[0]), type: "MultiPolygon" },
id: uUToUpgrade.find((item) => item.originalId === urbanUnity)?.id || ids[index],
level: 'urbanUnity',
nameFr: allUu[urbanUnity].uucr_nom,
originalId: urbanUnity,
})).map((geo) => (
db.collection(MONGO_TARGET_COLLECTION_NAME).updateOne({ originalId: geo.originalId }, { $set: geo }, { upsert: true })
));

await Promise.all(promises);
}

console.log('--- START ---');
await treatment();
await client.close();
console.log('--- END ---');


// {
// uu_code: {
// list: [
// {
// code_com: "ghjhg"
// geometry: {}
// }
// ],
// uucr_nom:
// }
// }


// urbanUnity
Loading

0 comments on commit be251d3

Please sign in to comment.