dataset: associate dataset directories to schemas

parent 1bbae159
......@@ -347,3 +347,69 @@ export const datasets: Datasets = {
},
],
}
const directory2schema: any = {
'.*Agenda_.*': 'reunion',
'.*Scrutins_.*': 'scrutin',
'.*Amendements_.*': 'amendement',
'.*Dossiers_Legislatifs_.*/documents': 'document',
'.*Dossiers_Legislatifs_.*/dossiers': 'dossier',
'.*acteurs_mandats_organes/organes': 'organe',
'.*acteurs_mandats_organes/acteurs': 'acteur',
'.*AMO30_tous_acteurs_tous_mandats_tous_organes_historique/organes': 'organe',
'.*AMO30_tous_acteurs_tous_mandats_tous_organes_historique/acteurs': 'acteur',
}
export function getDatasets(): any {
return [
'Agenda_XIV',
'Agenda_XV',
'Scrutins_XIV',
'Scrutins_XV',
'Amendements_XIV',
'Amendements_XV',
'Dossiers_Legislatifs_XV',
'Dossiers_Legislatifs_XIV',
'AMO30_tous_acteurs_tous_mandats_tous_organes_historique',
'acteurs_mandats_organes',
]
}
export function validDataset(dir: any): any {
for (const dataset of getDatasets()) {
if (dir.includes(dataset))
return true
}
return false
}
export function getSchemas(): any {
return Object.values(directory2schema)
}
export function datasetDirectorySchema(dataset: string): any {
let directories: any
if (dataset.includes('Dossiers_Legislatifs_')) {
directories = [
`${dataset}/documents`,
`${dataset}/dossiers`,
]
} else if(dataset.includes('acteurs_mandats_organes') ||
dataset.includes('AMO30_tous_acteurs_tous_mandats_tous_organes_historique')) {
directories = [
`${dataset}/organes`,
`${dataset}/acteurs`,
]
} else {
directories = [ dataset ]
}
let results = []
for (const directory of directories) {
for (const re of Object.keys(directory2schema)) {
if (new RegExp(re).exec(directory))
results.push([`${directory}/**/*.json`, directory2schema[re]])
}
}
return results
}
import { assert } from "chai"
import { datasetDirectorySchema, getSchemas, getDatasets } from "../src/datasets"
suite("dataset")
test("#datasetDirectorySchema", async function() {
assert.deepEqual(datasetDirectorySchema('parent/Agenda_XV'), [['parent/Agenda_XV/**/*.json', 'reunion']])
assert.deepEqual(datasetDirectorySchema('parent/Dossiers_Legislatifs_XV'), [["parent/Dossiers_Legislatifs_XV/documents/**/*.json", 'document'], ["parent/Dossiers_Legislatifs_XV/dossiers/**/*.json", 'dossier']])
for (const dataset of ['acteurs_mandats_organes', 'AMO30_tous_acteurs_tous_mandats_tous_organes_historique'])
assert.deepEqual(datasetDirectorySchema(dataset), [[`${dataset}/organes/**/*.json`, 'organe'], [`${dataset}/acteurs/**/*.json`, 'acteur']])
})
test("#getSchemas", async function() {
assert(getSchemas().includes('dossier'))
})
test("#getDatasets", async function() {
const datasets = getDatasets()
assert(datasets.length > 1)
for (const dataset of datasets)
assert(datasetDirectorySchema(dataset).length > 0)
})
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment