...
 
Commits (59)
......@@ -5,34 +5,6 @@ default:
paths:
- node_modules/
before_script:
## Install ssh-agent if not already installed, it is required by Docker.
## (change apt-get to yum if you use an RPM-based image)
- "which ssh-agent || ( apt update -y && apt install openssh-client -y )"
## Run ssh-agent (inside the build environment)
- eval $(ssh-agent -s)
## Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
## We're using tr to fix line endings which makes ed25519 keys work
## without extra base64 encoding.
## https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
## Create the SSH directory and give it the right permissions
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
## Accept the SSH host keys of git.en-root.org.
- echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
## Set the Git user name and email.
- git config --global user.email "tricoteuses@tricoteuses.fr"
- git config --global user.name "Tricoteuses données Assemblée"
- npm install
variables:
## This variable can be overridden at pipeline execution time.
## Possible values (when several values separate them with a space):
......@@ -43,6 +15,9 @@ variables:
## - Scrutins
## - All
CATEGORIES: All
## Ensure that all the Git tags are retrieved.
## https://docs.gitlab.com/ee/user/project/pipelines/settings.html#git-shallow-clone
GIT_DEPTH: 0
## Needed for storing locale dates
TZ: Europe/Paris
......@@ -108,11 +83,9 @@ qa:
stage: qa
rules:
- if: '$CI_SERVER_HOST == "git.en-root.org" && $CI_PROJECT_PATH == "tricoteuses/tricoteuses-assemblee" && $CATEGORIES == "All"'
before_script:
- /bin/true
script:
- set -x
- for project in AMO30_tous_acteurs_tous_mandats_tous_organes_historique_nettoye Dossiers_Legislatifs_XIV_nettoye Dossiers_Legislatifs_XV_nettoye Amendements_XV_nettoye Agenda_XIV_nettoye Agenda_XV_nettoye Scrutins_XIV_nettoye Scrutins_XV_nettoye ; do
- for project in AMO30_tous_acteurs_tous_mandats_tous_organes_historique_nettoye Dossiers_Legislatifs_XIV_nettoye Dossiers_Legislatifs_XV_nettoye Amendements_XV_nettoye Agenda_XV_nettoye Scrutins_XIV_nettoye Scrutins_XV_nettoye ; do
- curl --request POST --form "token=$DATA_TRICOTEUSES_FR_TRIGGER_TOKEN" --form ref=master --form "variables[PROJECT]=$project" https://git.en-root.org/api/v4/projects/59/trigger/pipeline # tricoteuses/tricoteuses-assemblee-qa
- done
- curl --request POST --form "token=$DATA_SITE_TRIGGER_TOKEN" --form ref=master https://git.en-root.org/api/v4/projects/9/trigger/pipeline # tricoteuses/data-site
......@@ -40,7 +40,7 @@ npx babel-node --extensions ".ts" -- src/scripts/retrieve_senateurs_photos.ts --
## Retrieval of documents from Assemblée nationale's website
```bash
npx babel-node --extensions ".ts" -- src/scripts/retrieve_textes_lois.ts ../assemblee-data/
npx babel-node --extensions ".ts" -- src/scripts/retrieve_documents.ts --textes ../data/assemblee-textes ../data/assemblee-nettoye/Dossiers_Legislatifs_XV_nettoye/documents/**/*.json
```
## Test loading everything in memory
......
{
"name": "@tricoteuses/assemblee",
"version": "0.16.9",
"version": "0.16.11",
"description": "Retrieve, clean up & handle French Assemblée nationale's open data",
"keywords": [
"Assemblée nationale",
......@@ -33,8 +33,8 @@
"coverage": "nyc -e .ts -x \"**/*.test.ts\" npm run test",
"coverage-lcov": "nyc -r lcov -e .ts -x \"**/*.test.ts\" npm run test",
"prepublishOnly": "npm run build",
"prettier": "npx prettier --write \"src/*.ts\" \"src/**/*.ts\"",
"test": "NODE_ICU_DATA=node_modules/full-icu mocha --require ts-node/register --ui qunit 'tests/**/*.test.ts'",
"prettier": "npx prettier --write 'src/**/*.ts' 'tests/**/*.test.ts'",
"test": "NODE_ICU_DATA=node_modules/full-icu mocha --timeout 10000 --require ts-node/register --ui qunit '**/*.test.ts'",
"type-check": "tsc --noEmit",
"type-check:watch": "npm run type-check -- --watch"
},
......@@ -90,6 +90,7 @@
"chai": "^4.2.0",
"core-js": "^3.6.4",
"eslint": "^6.7.2",
"fetch-mock": "^9.1.1",
"mocha": "^7.1.0",
"nyc": "^15.0.0",
"prettier": "^1.19.1",
......
......@@ -10,17 +10,19 @@ export class BasePlugin {
this.name = name
}
fix(_content: any, _filename: any): any {
async preFix(): Promise<any> {}
async fix(_content: any, _filename: any): Promise<any> {
return null
}
preCheck(): any {}
async preCheck(): Promise<any> {}
check(_content: any, _filename: any): any {
async check(_content: any, _filename: any): Promise<any> {
return null
}
postCheck(_analysis: any): any {}
async postCheck(_analysis: any): Promise<any> {}
}
export class SchemaBugs {
......@@ -34,20 +36,30 @@ export class SchemaBugs {
get(schema: any): any {
this.plugins = []
for (const file of getFiles([`src/bugs/${schema}-*.ts`])) {
const bugsDir = this.options["bugs-dir"] || "."
const files = getFiles([`${bugsDir}/src/bugs/${schema}-*.ts`]).filter(
(f: any) => this.options.bug == undefined || f.includes(this.options.bug),
)
for (const file of files) {
const module = require(`../${file}`)
this.plugins.push(new module["Plugin"](this.options))
}
}
fix(filenames: any): any {
async preFix(): Promise<any> {
for (const plugin of this.plugins) await plugin.preFix()
}
async fix(filenames: any): Promise<any> {
let results: any = {}
for (const plugin of this.plugins) {
for (const filename of filenames) {
const content = load(filename)
const result = plugin.fix(content, filename)
const result = await plugin.fix(content, filename)
results[plugin.name] = result
if (result == true) write(content, filename)
if (this.options.debug)
console.log(`${plugin.name} ${filename} ${result ? "FIXED" : "OK"}`)
}
if (this.options.commit) {
git.commit(this.options.commit, `Fix ${plugin.name}`)
......@@ -56,18 +68,18 @@ export class SchemaBugs {
return results
}
preCheck() {
for (const plugin of this.plugins) plugin.preCheck()
async preCheck(): Promise<void> {
for (const plugin of this.plugins) await plugin.preCheck()
}
check(content: any, filename: any): any {
async check(content: any, filename: any): Promise<any> {
let results: any = {}
for (const plugin of this.plugins)
results[plugin.name] = plugin.check(content, filename)
results[plugin.name] = await plugin.check(content, filename)
return results
}
postCheck(analysis: any): any {
for (const plugin of this.plugins) plugin.postCheck(analysis)
async postCheck(analysis: any): Promise<any> {
for (const plugin of this.plugins) await plugin.postCheck(analysis)
}
}
import { BasePlugin } from "../bugs"
export class Plugin extends BasePlugin {
constructor(options: any) {
super(options, "acteur-00010")
}
static ok(url: string): boolean {
return url == undefined || /^https:\/\//.test(url)
}
async check(acteur: any, _filename: any): Promise<any> {
const uriHatvp = acteur.uriHatvp
return {
status: Plugin.ok(uriHatvp) ? "ok" : "will-be-fixed",
info: [`uriHatvp == ${uriHatvp}`],
}
}
async fix(acteur: any, _filename: any): Promise<any> {
const uriHatvp = acteur.uriHatvp
if (Plugin.ok(uriHatvp)) {
return false
} else {
delete acteur.uriHatvp
return true
}
}
}
{
"uid": "PA1001",
"uriHatvp": "/tribun/resources/html/defautDeclarationActeur.html"
}
{
"uid": "PA1001",
"uriHatvp": "https://www.hatvp.fr/pages_nominatives/meizonnet-nicolas-17348"
}
import { load } from "../../file_systems"
import { assert } from "chai"
import { Plugin } from "../acteur-00010"
const id = "00010"
suite(`Plugin${id}`)
test(`#Plugin${id}`, async function() {
const plugin = new Plugin({})
const d = `src/bugs/acteur-${id}`
{
const f = `${d}/fail.json`
const content = load(f)
assert("uriHatvp" in content)
const result = await plugin.check(content, f)
assert.deepEqual(result["status"], "will-be-fixed", f)
assert.equal(await plugin.fix(content, f), true, f)
assert(!("uriHatvp" in content))
}
{
const f = `${d}/ok.json`
const content = load(f)
const result = await plugin.check(content, f)
assert.deepEqual(result["status"], "ok", f)
assert.equal(await plugin.fix(content, f), false, f)
}
{
const f = `${d}/ok-missing.json`
const content = load(f)
const result = await plugin.check(content, f)
assert.deepEqual(result["status"], "ok", f)
assert.equal(await plugin.fix(content, f), false, f)
}
})
......@@ -3,8 +3,9 @@ import { HTMLElement, parse, TextNode } from "node-html-parser"
import fs from "fs-extra"
import { getFiles } from "../file_systems"
import { BasePlugin } from "../bugs"
import { masterUrl } from "../datasets"
const remapJo: any = {
export let remapJo: any = {
RUANR5L15S2017IDS20667: "20172002",
RUANR5L15S2018IDS20817: "20180086",
RUANR5L15S2018IDS20864: "20180121",
......@@ -35,10 +36,11 @@ export class Plugin extends BasePlugin {
filename2date: any
constructor(options: any) {
super(options, "reunion-00002")
super(options, "agenda-00002")
}
preCheck(): any {
async preCheck(): Promise<void> {
if (this.options.cr == undefined) throw "--cr is required"
this.filename2date = {}
this.cr2filenames = {}
this.date2crs = {}
......@@ -73,22 +75,28 @@ export class Plugin extends BasePlugin {
}
}
check(reunion: any, filename: any): any {
static comptesRendusUrls(filename: string): string {
const re = "^.*/comptes-rendus/(.*)"
const match = new RegExp(re).exec(filename)
if (match == null) return filename
const file = match[1]
return `[${file}](https://git.en-root.org/tricoteuses/html/comptes-rendus/-/tree/master/${file})`
}
async check(agenda: any, filename: any): Promise<any> {
if (
reunion.xsiType != "seance_type" ||
reunion.lieu.code != "AN" ||
reunion.cycleDeVie.etat == "Supprimé" ||
reunion.cycleDeVie.etat == "Annulé"
agenda.xsiType != "seance_type" ||
agenda.lieu.code != "AN" ||
agenda.cycleDeVie.etat == "Supprimé" ||
agenda.cycleDeVie.etat == "Annulé"
)
return null
if (this.options.verbose) console.log(`reunion-00002: ${filename}`)
if (this.options.verbose) console.log(`agenda-00002: ${filename}`)
let status = "ok"
let info: any = []
const dateSeance = new Date(
reunion.identifiants.dateSeance.substring(0, 10),
)
const dateSeance = new Date(agenda.identifiants.dateSeance.substring(0, 10))
const year = dateSeance.toLocaleDateString("fr", { year: "numeric" })
const day = dateSeance.toLocaleDateString("fr", { day: "2-digit" })
const month = dateSeance.toLocaleDateString("fr", { month: "long" })
......@@ -98,8 +106,8 @@ export class Plugin extends BasePlugin {
let idJo
if (
reunion.identifiants == undefined ||
reunion.identifiants.idJo == undefined
agenda.identifiants == undefined ||
agenda.identifiants.idJo == undefined
) {
const week = 7 * 24 * 60 * 60 * 1000
if (dateSeance >= new Date(new Date().getTime() - week)) {
......@@ -108,15 +116,15 @@ export class Plugin extends BasePlugin {
idJo = undefined
}
} else {
idJo = reunion.identifiants.idJo
idJo = agenda.identifiants.idJo
}
if (reunion.uid in remapJo) {
if (idJo == remapJo[reunion.uid]) {
if (agenda.uid in remapJo) {
if (idJo == remapJo[agenda.uid]) {
info.push(`idJo already is ${idJo}`)
status = "already-fixed"
} else {
idJo = remapJo[reunion.uid]
idJo = remapJo[agenda.uid]
status = "will-be-fixed"
}
}
......@@ -138,7 +146,9 @@ export class Plugin extends BasePlugin {
if (dateSeanceString != this.cr2date[cr]) {
info.push(
`${cr} meta QUANTIEME_SEANCE contains ${this.cr2date[cr]} instead of the expected ${dateSeanceString}`,
`${Plugin.comptesRendusUrls(cr)} meta QUANTIEME_SEANCE contains ${
this.cr2date[cr]
} instead of the expected ${dateSeanceString}`,
)
status = "needs-fixing"
}
......@@ -158,16 +168,20 @@ export class Plugin extends BasePlugin {
for (const cr of this.date2crs[date]) {
if (this.cr2filenames[cr].length > 0)
candidates.push(
`${cr} is not a candidate because it is referenced by ${this.cr2filenames[cr]}`,
`${Plugin.comptesRendusUrls(
cr,
)} is not a candidate because it is referenced by ${masterUrl(
this.cr2filenames[cr],
)}`,
)
else candidates.push(`${cr} is a candidate`)
else candidates.push(`${Plugin.comptesRendusUrls(cr)} is a candidate`)
}
return candidates
}
postCheck(analysis: any): any {
async postCheck(analysis: any): Promise<any> {
for (const filename of Object.keys(analysis)) {
const result = analysis[filename]["reunion-00002"]
const result = analysis[filename]["agenda-00002"]
if (result == null || result["status"] != "needs-fixing") continue
const info = result["info"].join("")
if (
......@@ -179,20 +193,20 @@ export class Plugin extends BasePlugin {
}
}
fix(reunion: any): any {
if (!(reunion.uid in remapJo)) {
async fix(agenda: any): Promise<any> {
if (!(agenda.uid in remapJo)) {
return null
}
const idJo = remapJo[reunion.uid]
const idJo = remapJo[agenda.uid]
const numSeanceJo = String(Number(idJo.substring(5)))
if (
reunion.identifiants.idJo == idJo &&
reunion.identifiants.numSeanceJo == numSeanceJo
agenda.identifiants.idJo == idJo &&
agenda.identifiants.numSeanceJo == numSeanceJo
)
return false
reunion.identifiants.idJo = idJo
reunion.identifiants.numSeanceJo = numSeanceJo
agenda.identifiants.idJo = idJo
agenda.identifiants.numSeanceJo = numSeanceJo
return true
}
}
import { load } from "../../file_systems"
import { assert } from "chai"
import { Plugin, remapJo } from "../agenda-00002"
const id = "00002"
suite(`Plugin${id}`)
test(`#Plugin${id}`, async function() {
Object.assign(remapJo, {
RUANR5L15S2020IDS22089: "20200140",
})
const plugin = new Plugin({ cr: "src/bugs/agenda-00002" })
const d = `src/bugs/agenda-${id}`
for (const expectedStatus of [
"will-be-fixed",
"already-fixed",
"ok",
"needs-fixing",
]) {
const f = `${d}/${expectedStatus}.json`
const content = load(f)
await plugin.preCheck()
const r = await plugin.check(content, f)
assert.equal(
r["status"],
expectedStatus,
`${f} returns ${r["status"]} ${r["info"]}`,
)
await plugin.postCheck({ f: r })
assert.equal(
r["status"],
expectedStatus,
`${f} returns ${r["status"]} ${r["info"]}`,
)
}
{
const f = `${d}/null.json`
const content = load(f)
await plugin.preCheck()
const r = await plugin.check(content, f)
assert.equal(r, null, f)
}
})
import { BasePlugin } from "../bugs"
export let CANCELED = [
"RUANR5L15S2020IDS21979",
"RUANR5L15S2020IDS22027",
"RUANR5L15S2020IDS21997",
"RUANR5L15S2020IDS21968",
"RUANR5L15S2019IDS21385",
"RUANR5L15S2018IDS20750",
]
export class Plugin extends BasePlugin {
constructor(options: any) {
super(options, "agenda-00008")
}
async check(agenda: any, _filename: any): Promise<any> {
if (CANCELED.indexOf(agenda.uid) < 0) return null
const etat = agenda.cycleDeVie.etat
const fixed = etat == "Supprimé"
return {
status: fixed ? "already-fixed" : "will-be-fixed",
info: [`agenda.cycleDeVie.etat == ${etat}`],
}
}
async fix(agenda: any, _filename: any): Promise<any> {
if (CANCELED.indexOf(agenda.uid) < 0) return null
if (agenda.cycleDeVie.etat == "Confirmé") {
agenda.cycleDeVie.etat = "Supprimé"
return true
} else {
return false
}
}
}
{
"uid": "RUANR5L15S2018IDS20750",
"cycleDeVie": {
"etat": "Confirmé"
}
}
{
"uid": "RUANR5L15S2018IDS20750",
"cycleDeVie": {
"etat": "Supprimé"
}
}
{
"uid": "RUANR5L15S2018IDS6666",
"cycleDeVie": {
"etat": "Supprimé"
}
}
import { load } from "../../file_systems"
import { assert } from "chai"
import { Plugin, CANCELED } from "../agenda-00008"
const id = "00008"
suite(`Plugin${id}`)
test(`#Plugin${id}`, async function() {
CANCELED.splice(0, CANCELED.length, "RUANR5L15S2018IDS20750")
const plugin = new Plugin({})
const d = `src/bugs/agenda-${id}`
{
const f = `${d}/fail.json`
const content = load(f)
const result = await plugin.check(content, f)
assert.deepEqual(result["status"], "will-be-fixed", f)
assert.equal(await plugin.fix(content, f), true, f)
}
{
const f = `${d}/ok.json`
const content = load(f)
const result = await plugin.check(content, f)
assert.deepEqual(result["status"], "already-fixed", f)
assert.equal(await plugin.fix(content, f), false, f)
}
{
const f = `${d}/other.json`
const content = load(f)
assert.deepEqual(await plugin.check(content, f), null, f)
assert.equal(await plugin.fix(content, f), null, f)
}
})
import fetch from "node-fetch"
import { BasePlugin } from "../bugs"
// https://www.wikidata.org/wiki/Wikidata:WikiProject_France/Assembl%C3%A9e_Nationale#Legislative_session
const url =
"https://query.wikidata.org/sparql?query=SELECT%20DISTINCT%20%3Fsession%20%3Fstart_time%20%3Fend_time%20%3Flabel%20%3Ffollows%20%3Ffollowed_by%20WHERE%20%7B%0A%0A%20%20%20%3Fsession%20wdt%3AP31%2Fwdt%3AP279*%20wd%3AQ87337481.%0A%20%20%20%3Fsession%20wdt%3AP580%20%3Fstart_time.%20%0A%20%20%20OPTIONAL%20%7B%20%3Fsession%20wdt%3AP582%20%3Fend_time%20%7D.%0A%20%20%20OPTIONAL%20%7B%20%3Fsession%20wdt%3AP155%20%3Ffollows%20%7D.%0A%20%20%20OPTIONAL%20%7B%20%3Fsession%20wdt%3AP156%20%3Ffollowed_by%20%7D.%0A%20%20%20OPTIONAL%20%7B%20%3Fsession%20rdfs%3Alabel%20%3Flabel%20filter%20(lang(%3Flabel)%20%3D%20%22fr%22)%20.%7D%0A%7D"
function compareString(a: string, b: string) {
if (a < b) return -1
if (a > b) return 1
return 0
}
export class Plugin extends BasePlugin {
sessions: any
fetch: any
constructor(options: any) {
super(options, "agenda-00011")
this.fetch = fetch
}
async preFix(): Promise<any> {
return this.setSessions()
}
async preCheck(): Promise<any> {
return this.setSessions()
}
async setSessions(): Promise<any> {
const sessions = (await this.fetchSessions()).sort((a: any, b: any) =>
compareString(a["start_time"]["value"], b["start_time"]["value"]),
)
this.sessions = sessions.map((s: any) => {
return {
label: s["label"]["value"].replace(/\s+en france/i, ""),
debut: s["start_time"]["value"].substr(0, 10),
fin: s["end_time"]["value"].substr(0, 10),
}
})
}
async get(url: string, headers: any): Promise<any> {
return this.fetch(url, headers)
}
async fetchSessions(): Promise<any> {
const response = await this.get(url, {
headers: { Accept: "application/json" },
})
if (!response.ok) throw response
const payload = await response.json()
return payload["results"]["bindings"]
}
lookupSession(debut: string): any {
const max = this.sessions.length - 1
const min = 0
let h = max
let m
let l = min
let session
let ok
do {
m = Math.floor((l + h) / 2)
session = this.sessions[m]
ok = session.debut <= debut && debut <= session.fin
if (!ok) {
if (l == m && m == h)
// in between sessions
return null
if (debut < session.debut) {
if (m <= min) return null
h = m
} else if (debut > session.fin) {
if (m >= max) return null
l = l == m ? h : m
}
}
} while (!ok)
return session
}
inScope(agenda: any) {
if (
agenda.xsiType != "seance_type" ||
(agenda.lieu !== undefined &&
agenda.lieu.code != "CG" &&
agenda.lieu.code != "AN")
)
return false
if (agenda.cycleDeVie.etat != "Confirmé") return false
return true
}
async check(agenda: any, _filename: any): Promise<any> {
if (!this.inScope(agenda)) return null
const debut = agenda.timestampDebut.substr(0, 10)
const session = this.lookupSession(debut)
if (session == null) {
return { status: "needs-fixing", info: [`${debut} is not in a session`] }
} else {
return { status: "ok", info: [""] }
}
}
async fix(agenda: any, _filename: any): Promise<any> {
let status
if ("sessionRef" in agenda) {
// sessionRef is undefined and must be removed
delete agenda.sessionRef
status = true
} else {
// if there is no sessionRef to remove, file which is not in scope does not need to be rewritten
status = null
}
if (!this.inScope(agenda)) return status
if ("session" in agenda) delete agenda.session
const session = this.lookupSession(agenda.timestampDebut.substr(0, 10))
if (session == null) {
return null
} else {
agenda.session = session
return true
}
}
}
{
"xsiType": "seance_type",
"uid": "UID",
"cycleDeVie": {
"etat": "Confirmé"
},
"timestampDebut": "2017-06-20T18:00:00.000+02:00"
}
{
"xsiType": "seance_type",
"uid": "UID",
"cycleDeVie": {
"etat": "Confirmé"
},
"timestampDebut": "2020-01-12T18:00:00.000+02:00"
}
{
"xsiType": "reunionCommission_type",
"uid": "UID",
"cycleDeVie": {
"etat": "Confirmé"
},
"timestampDebut": "2017-06-20T18:00:00.000+02:00"
}
{
"xsiType": "seance_type",
"uid": "UID",
"cycleDeVie": {
"etat": "Confirmé"
},
"timestampDebut": "2020-01-12T18:00:00.000+02:00",
"sessionRef": ""
}
import { load } from "../../file_systems"
import { assert } from "chai"
import { Plugin } from "../agenda-00011"
import fetchMock from "fetch-mock"
const id = "00011"
suite(`Plugin${id}`)
test(`#Plugin${id}FindSession`, async function() {
const plugin = new Plugin({})
const sessions = [
{
debut: "2019-09-10",
fin: "2019-09-27",
},
{
debut: "2019-10-01",
fin: "2020-06-30",
},
{
debut: "2020-06-31",
fin: "2020-07-31",
},
{
debut: "2020-08-01",
fin: "2020-08-31",
},
{
debut: "2020-09-01",
fin: "2020-09-30",
},
]
plugin.sessions = sessions.slice(0, 2)
// between sessions
assert.strictEqual(plugin.lookupSession("2019-09-28"), null)
assert.strictEqual(plugin.lookupSession("2017-01-01"), null)
assert.deepEqual(plugin.lookupSession("2019-09-15"), plugin.sessions[0])
assert.deepEqual(plugin.lookupSession("2020-02-02"), plugin.sessions[1])
assert.strictEqual(plugin.lookupSession("2022-01-01"), null)
plugin.sessions = sessions.slice(0, 1)
assert.strictEqual(plugin.lookupSession("2017-01-01"), null)
assert.deepEqual(plugin.lookupSession("2019-09-15"), plugin.sessions[0])
assert.strictEqual(plugin.lookupSession("2020-02-02"), null)
assert.strictEqual(plugin.lookupSession("2022-01-01"), null)
plugin.sessions = sessions
assert.strictEqual(plugin.lookupSession("2017-01-01"), null)
assert.deepEqual(plugin.lookupSession("2019-09-15"), plugin.sessions[0])
assert.deepEqual(plugin.lookupSession("2020-02-02"), plugin.sessions[1])
assert.deepEqual(plugin.lookupSession("2020-09-01"), plugin.sessions[4])
assert.deepEqual(plugin.lookupSession("2020-09-30"), plugin.sessions[4])
assert.strictEqual(plugin.lookupSession("2022-01-01"), null)
})
test(`#Plugin${id}Load`, async function() {
const plugin = new Plugin({})
const d = `src/bugs/agenda-${id}`
const wikidata = load(`${d}/wikidata.json`)
plugin.fetch = fetchMock
.sandbox()
.mock(/query\.wikidata\.org/, { body: wikidata, status: 200 })
await plugin.preFix()
const expected = [
{
debut: "2019-09-10",
fin: "2019-09-27",
label: "SESSION02",
},
{
debut: "2019-10-01",
fin: "2020-06-30",
label: "SESSION01",
},
]
assert.deepEqual(plugin.sessions, expected)
fetchMock.restore()
})
test(`#Plugin${id}Fix`, async function() {
const plugin = new Plugin({})
const d = `src/bugs/agenda-${id}`
const wikidata = load(`${d}/wikidata.json`)
plugin.fetch = fetchMock
.sandbox()
.mock(/query\.wikidata\.org/, { body: wikidata, status: 200 })
await plugin.preFix()
assert(plugin.sessions != undefined)
{
const f = `${d}/ok.json`
const content = load(f)
assert("sessionRef" in content, content)
assert.equal(await plugin.fix(content, f), true, f)
assert(!("sessionRef" in content), content)
assert(content.session.label, "SESSION01")
}
{
const f = `${d}/nosessionref.json`
const content = load(f)
assert.equal(await plugin.fix(content, f), true, f)
assert.equal(content.session.label, "SESSION01")
}
{
const f = `${d}/session.json`
const content = load(f)
assert.equal(content.session.label, "SOMETHING")
assert.equal(await plugin.fix(content, f), true, f)
assert.equal(content.session.label, "SESSION01")
}
{
const f = `${d}/not_seance_type.json`
const content = load(f)
assert.equal(await plugin.fix(content, f), null, f)
}
fetchMock.restore()
})
test(`#Plugin${id}Check`, async function() {
const plugin = new Plugin({})
const d = `src/bugs/agenda-${id}`
const wikidata = load(`${d}/wikidata.json`)
plugin.fetch = fetchMock
.sandbox()
.mock(/query\.wikidata\.org/, { body: wikidata, status: 200 })
await plugin.preCheck()
assert(plugin.sessions != undefined)
{
const f = `${d}/ok.json`
const content = load(f)
const result = await plugin.check(content, f)
assert.deepEqual(result, { status: "ok", info: [""] }, f)
}
{
const f = `${d}/badtimestamp.json`
const content = load(f)
const result = await plugin.check(content, f)
assert.deepEqual(
result,
{ status: "needs-fixing", info: ["2017-06-20 is not in a session"] },
f,
)
}
fetchMock.restore()
})
test(`#Plugin${id}FetchFail`, async function() {
const plugin = new Plugin({})
plugin.fetch = fetchMock.sandbox().mock(/query\.wikidata\.org/, 500)
let status = "unexpected"
await plugin
.preFix()
.then(() => (status = "exception not thrown"))
.catch(e => (status = String(e.status)))
assert.equal(status, "500")
fetchMock.restore()
})
{
"xsiType": "seance_type",
"uid": "UID",
"cycleDeVie": {
"etat": "Confirmé"
},
"timestampDebut": "2020-01-12T18:00:00.000+02:00",
"session": {
"label": "SOMETHING"
}
}
{
"head" : {
"vars" : [ "session", "start_time", "end_time", "label", "follows", "followed_by" ]
},
"results" : {
"bindings" : [ {
"session" : {
"type" : "uri",
"value" : "http://www.wikidata.org/entity/Q87343249"
},
"label" : {
"xml:lang" : "fr",
"type" : "literal",
"value" : "SESSION01 en France"
},
"start_time" : {
"datatype" : "http://www.w3.org/2001/XMLSchema#dateTime",
"type" : "literal",
"value" : "2019-10-01T00:00:00Z"
},
"end_time" : {
"datatype" : "http://www.w3.org/2001/XMLSchema#dateTime",
"type" : "literal",
"value" : "2020-06-30T00:00:00Z"
},
"follows" : {
"type" : "uri",
"value" : "http://www.wikidata.org/entity/Q87344260"
}
}, {
"session" : {
"type" : "uri",
"value" : "http://www.wikidata.org/entity/Q87344260"
},
"label" : {
"xml:lang" : "fr",
"type" : "literal",
"value" : "SESSION02"
},
"start_time" : {
"datatype" : "http://www.w3.org/2001/XMLSchema#dateTime",
"type" : "literal",
"value" : "2019-09-10T00:00:00Z"
},
"end_time" : {
"datatype" : "http://www.w3.org/2001/XMLSchema#dateTime",
"type" : "literal",
"value" : "2019-09-27T00:00:00Z"
},
"followed_by" : {
"type" : "uri",
"value" : "http://www.wikidata.org/entity/Q87343249"
}
} ]
}
}
{
"xsiType": "seance_type",
"uid": "UID",
"cycleDeVie": {
"etat": "Confirmé"
},
"timestampDebut": "2017-06-20T18:00:00.000+02:00",
"sessionRef": ""
}
import { createHash } from "crypto"
import fs from "fs-extra"
import path from "path"
......@@ -349,36 +350,37 @@ export const datasets: Datasets = {
}
const directory2schema: any = {
'.*Agenda_.*': 'reunion',
'.*Scrutins_.*': 'scrutin',
'.*Amendements_.*': 'amendement',
'.*Dossiers_Legislatifs_.*/documents': 'document',
'.*Dossiers_Legislatifs_.*/dossiers': 'dossier',
'.*acteurs_mandats_organes/organes': 'organe',
'.*acteurs_mandats_organes/acteurs': 'acteur',
'.*AMO30_tous_acteurs_tous_mandats_tous_organes_historique/organes': 'organe',
'.*AMO30_tous_acteurs_tous_mandats_tous_organes_historique/acteurs': 'acteur',
".*Agenda_.*": "agenda",
".*Scrutins_.*": "scrutin",
".*Amendements_.*": "amendement",
".*Dossiers_Legislatifs_.*/documents": "document",
".*Dossiers_Legislatifs_.*/dossiers": "dossier",
".*acteurs_mandats_organes/organes": "organe",
".*acteurs_mandats_organes/acteurs": "acteur",
".*AMO30_tous_acteurs_tous_mandats_tous_organes_historique.*/organes":
"organe",
".*AMO30_tous_acteurs_tous_mandats_tous_organes_historique.*/acteurs":
"acteur",
}
export function getDatasets(): any {
return [
'Agenda_XIV',
'Agenda_XV',
'Scrutins_XIV',
'Scrutins_XV',
'Amendements_XIV',
'Amendements_XV',
'Dossiers_Legislatifs_XV',
'Dossiers_Legislatifs_XIV',
'AMO30_tous_acteurs_tous_mandats_tous_organes_historique',
'acteurs_mandats_organes',
"Agenda_XIV",
"Agenda_XV",
"Scrutins_XIV",
"Scrutins_XV",
"Amendements_XIV",
"Amendements_XV",
"Dossiers_Legislatifs_XV",
"Dossiers_Legislatifs_XIV",
"AMO30_tous_acteurs_tous_mandats_tous_organes_historique",
"acteurs_mandats_organes",
]
}
export function validDataset(dir: any): any {
for (const dataset of getDatasets()) {
if (dir.includes(dataset))
return true
if (dir.includes(dataset)) return true
}
return false
}
......@@ -389,27 +391,51 @@ export function getSchemas(): any {
export function datasetDirectorySchema(dataset: string): any {
let directories: any
if (dataset.includes('Dossiers_Legislatifs_')) {
directories = [
`${dataset}/documents`,
`${dataset}/dossiers`,
]
} else if(dataset.includes('acteurs_mandats_organes') ||
dataset.includes('AMO30_tous_acteurs_tous_mandats_tous_organes_historique')) {
directories = [
`${dataset}/organes`,
`${dataset}/acteurs`,
]
if (dataset.includes("Dossiers_Legislatifs_")) {
directories = [`${dataset}/documents`, `${dataset}/dossiers`]
} else if (
dataset.includes("acteurs_mandats_organes") ||
dataset.includes("AMO30_tous_acteurs_tous_mandats_tous_organes_historique")
) {
directories = [`${dataset}/organes`, `${dataset}/acteurs`]
} else {
directories = [ dataset ]
directories = [dataset]
}
let results = []
for (const directory of directories) {
for (const re of Object.keys(directory2schema)) {
if (new RegExp(re).exec(directory))
results.push([`${directory}/**/*.json`, directory2schema[re]])
results.push([`${directory}/**/*.json`, directory2schema[re]])
}
}
return results
}
export function fileDatasetUrl(filename: string): [string, string] | null {
const re = "^.*/(.*)_nettoye/(.*)"
const match = new RegExp(re).exec(filename)
if (match == null) return null
let [dataset, file] = [match[1], match[2]]
if (dataset == "AMO30_tous_acteurs_tous_mandats_tous_organes_historique")
dataset = "acteurs_mandats_organes"
return [dataset, file]
}
export function diffUpstream(filename: string): string {
const datasetUrl = fileDatasetUrl(filename)
if (datasetUrl == null) return filename
const [dataset, file] = datasetUrl
const sum = createHash("sha1")
.update(file)
.digest("hex")
.toString()
return `[${file}](https://git.en-root.org/tricoteuses/data.tricoteuses.fr/${dataset}/-/compare/upstream...master#${sum})`
}
export function masterUrl(filename: string): string {
const datasetUrl = fileDatasetUrl(filename)
if (datasetUrl == null) return filename
const [dataset, file] = datasetUrl
return `[${file}](https://git.en-root.org/tricoteuses/data.tricoteuses.fr/${dataset}/-/tree/master/${file})`
}
......@@ -36,14 +36,26 @@ export function write(something: any, filename: string): any {
})
}
export class GetFileNotFoundError {
message: string
constructor(message: string) {
this.message = message
}
}
export function getFiles(args: any): any {
let files: string[] = []
function _getFiles(fileOrPattern: any) {
if (glob.hasMagic(fileOrPattern)) {
const dataFiles = glob.sync(fileOrPattern, { cwd: process.cwd() })
if (dataFiles.length <= 0)
throw new GetFileNotFoundError(`${fileOrPattern} glob does not match`)
files = files.concat(dataFiles)
} else {
files.push(fileOrPattern)
if (!fs.existsSync(fileOrPattern))
throw new GetFileNotFoundError(`${fileOrPattern} file does not exist`)
}
}
args.forEach(_getFiles)
......
import { execSync } from "child_process"
export function run(repositoryDir: string, args: string): string {
return execSync(`git ${args}`, { cwd: repositoryDir })
.toString()
.trim()
const MAXBUFFER = 50 * 1024 * 1024
export function run(
repositoryDir: string,
args: string,
verbose?: boolean,
): string {
try {
if (verbose) console.log(`git -C ${repositoryDir} ${args}`)
const output = execSync(`git ${args}`, {
cwd: repositoryDir,
maxBuffer: MAXBUFFER,
})
.toString()
.trim()
if (verbose) console.log(output)
return output
} catch (childProcess) {
for (const output of ["stdout", "stderr"])
console.error(`${output}: ${childProcess[output]}`)
throw childProcess
}
}
export function test(repositoryDir: string, args: string): boolean {
export function test(
repositoryDir: string,
args: string,
verbose?: boolean,
): boolean {
try {
execSync(`git ${args}`, {
if (verbose) console.log(`git -C ${repositoryDir} ${args}`)
const output = execSync(`git ${args}`, {
cwd: repositoryDir,
stdio: ["ignore", "pipe", "pipe"],
maxBuffer: MAXBUFFER,
})
.toString()
.trim()
if (verbose) console.log(output)
return true
} catch (childProcess) {
if (childProcess.status != 0) return false
......@@ -25,6 +52,7 @@ export function commit(repositoryDir: string, message: string): boolean {
env: process.env,
encoding: "utf-8",
stdio: ["ignore", "ignore", "pipe"],
maxBuffer: MAXBUFFER,
})
try {
execSync(`git commit -m "${message}"`,