Commit 8326d40f authored by Pauline Léon's avatar Pauline Léon

Merge branch 'wip-agenda-2' into 'master'

bugs and validation CLI and infrastructure

See merge request tricoteuses/tricoteuses-assemblee!40
parents b003c8a6 90575d2a
Pipeline #6295 passed with stages
in 2 minutes and 50 seconds
......@@ -34,7 +34,7 @@
"coverage-lcov": "nyc -r lcov -e .ts -x \"**/*.test.ts\" npm run test",
"prepublishOnly": "npm run build",
"prettier": "npx prettier --write \"src/*.ts\" \"src/**/*.ts\"",
"test": "mocha --require ts-node/register --ui qunit tests/**/*.test.ts",
"test": "NODE_ICU_DATA=node_modules/full-icu mocha --require ts-node/register --ui qunit 'tests/**/*.test.ts'",
"type-check": "tsc --noEmit",
"type-check:watch": "npm run type-check -- --watch"
},
......@@ -44,7 +44,9 @@
"deep-object-diff": "^1.1.0",
"fs-extra": "^8.1.0",
"jsdom": "^16.2.0",
"full-icu": "^1.3.1",
"node-fetch": "^2.6.0",
"node-html-parser": "^1.2.7",
"node-stream-zip": "^1.9.1"
},
"devDependencies": {
......
import * as git from "./git"
import { load, write, getFiles } from "./file_systems"
export class BasePlugin {
options: any
name: string
constructor(options: any, name: string) {
this.options = options
this.name = name
}
fix(_content: any, _filename: any): any {
return null
}
preCheck(): any {}
check(_content: any, _filename: any): any {
return null
}
postCheck(_analysis: any): any {}
}
export class SchemaBugs {
options: any
plugins: any
constructor(options: any) {
this.options = options
this.get(this.options.schema)
}
get(schema: any): any {
this.plugins = []
for (const file of getFiles([`src/bugs/${schema}-*.ts`])) {
const module = require(`../${file}`)
this.plugins.push(new module["Plugin"](this.options))
}
}
fix(filenames: any): any {
let results: any = {}
for (const plugin of this.plugins) {
for (const filename of filenames) {
const content = load(filename)
const result = plugin.fix(content, filename)
results[plugin.name] = result
if (result == true) write(content, filename)
}
if (this.options.commit) {
git.commit(this.options.commit, `Fix ${plugin.name}`)
}
}
return results
}
preCheck() {
for (const plugin of this.plugins) plugin.preCheck()
}
check(content: any, filename: any): any {
let results: any = {}
for (const plugin of this.plugins)
results[plugin.name] = plugin.check(content, filename)
return results
}
postCheck(analysis: any): any {
for (const plugin of this.plugins) plugin.postCheck(analysis)
}
}
import { BasePlugin } from "../bugs"
function fixActeLegislatif(acteLegislatif: any): any {
const statutConclusion = acteLegislatif.statutConclusion
if (statutConclusion !== undefined) {
if (statutConclusion.famCode === "TSORTFnull") {
delete acteLegislatif.statutConclusion
return true
}
}
let actesLegislatifs = acteLegislatif.actesLegislatifs
if (actesLegislatifs !== undefined) {
for (const acteLegislatif of actesLegislatifs) {
if (fixActeLegislatif(acteLegislatif) == true) return true
}
}
return false
}
export class Plugin extends BasePlugin {
constructor(options: any, _name?: string) {
super(options, "dossier-00001")
}
fix(dossier: any, _filename: any): any {
if (dossier.uid != "DLR5L15N37831") {
return null
}
let actesLegislatifs = dossier.actesLegislatifs
for (const acteLegislatif of actesLegislatifs) {
if (fixActeLegislatif(acteLegislatif) == true) return true
}
return false
}
}
import glob from "glob"
import { HTMLElement, parse, TextNode } from "node-html-parser"
import fs from "fs-extra"
import { getFiles } from "../file_systems"
import { BasePlugin } from "../bugs"
const remapJo: any = {
RUANR5L15S2017IDS20667: "20172002",
RUANR5L15S2018IDS20817: "20180086",
RUANR5L15S2018IDS20864: "20180121",
RUANR5L15S2018IDS20981: "20180174",
RUANR5L15S2019IDS21507: "20190153",
RUANR5L15S2019IDS21608: "20190218",
RUANR5L15S2020IDS21840: "20200002",
RUANR5L15S2020IDS21841: "20200003",
RUANR5L15S2020IDS21859: "20200001",
RUANR5L15S2020IDS21970: "20200090",
RUANR5L15S2020IDS21983: "20200092",
RUANR5L15S2020IDS22019: "20200117",
RUANR5L15S2020IDS22020: "20200118",
RUANR5L15S2020IDS22021: "20200119",
RUANR5L15S2020IDS22054: "20200134",
RUANR5L15S2020IDS22055: "20200135",
RUANR5L15S2020IDS22056: "20200136",
RUANR5L15S2020IDS22057: "20200137",
RUANR5L15S2020IDS22087: "20200139",
RUANR5L15S2020IDS22089: "20200140",
RUANR5L15S2020IDS22106: "20200125",
}
export class Plugin extends BasePlugin {
cr2filenames: any
date2crs: any
cr2date: any
filename2date: any
constructor(options: any) {
super(options, "reunion-00002")
}
preCheck(): any {
this.filename2date = {}
this.cr2filenames = {}
this.date2crs = {}
this.cr2date = {}
for (const cr of getFiles([`${this.options.cr}/*.asp`])) {
this.cr2filenames[cr] = []
const content: string = fs.readFileSync(cr, { encoding: "utf-8" })
type ParseResult =
| (TextNode & { valid: boolean })
| (HTMLElement & { valid: boolean })
const doc: ParseResult = parse(content)
const root = (doc as unknown) as HTMLElement
let found = false
for (const meta of root.querySelectorAll("meta")) {
if (meta.attributes["name"] == "QUANTIEME_SEANCE") {
const date = meta.attributes["content"]
.split(" ")
.splice(-3)
.join(" ")
if (!(date in this.date2crs)) this.date2crs[date] = []
this.date2crs[date].push(cr)
this.cr2date[cr] = date
found = true
break
}
}
if (!found) {
console.error(`${cr} is missing the meta name="QUANTIEME_SEANCE"`)
}
}
}
check(reunion: any, filename: any): any {
if (
reunion.xsiType != "seance_type" ||
reunion.lieu.code != "AN" ||
reunion.cycleDeVie.etat == "Supprimé" ||
reunion.cycleDeVie.etat == "Annulé"
)
return null
if (this.options.verbose) console.log(`reunion-00002: ${filename}`)
let status = "ok"
let info: any = []
const dateSeance = new Date(
reunion.identifiants.dateSeance.substring(0, 10),
)
const year = dateSeance.toLocaleDateString("fr", { year: "numeric" })
const day = dateSeance.toLocaleDateString("fr", { day: "2-digit" })
const month = dateSeance.toLocaleDateString("fr", { month: "long" })
const dateSeanceString = `${day} ${month} ${year}`
this.filename2date[filename] = dateSeanceString
let idJo
if (
reunion.identifiants == undefined ||
reunion.identifiants.idJo == undefined
) {
const week = 7 * 24 * 60 * 60 * 1000
if (dateSeance >= new Date(new Date().getTime() - week)) {
return null
} else {
idJo = undefined
}
} else {
idJo = reunion.identifiants.idJo
}
if (reunion.uid in remapJo) {
if (idJo == remapJo[reunion.uid]) {
info.push(`idJo already is ${idJo}`)
status = "already-fixed"
} else {
idJo = remapJo[reunion.uid]
status = "will-be-fixed"
}
}
if (idJo == undefined) {
return {
status: "needs-fixing",
info: ["idJo field is missing"],
}
}
const pattern = `${this.options.cr}/*_${idJo}.asp`
const expanded = glob.sync(pattern)
let cr
if (expanded.length > 0) cr = expanded[0]
else cr = undefined
if (cr != undefined) {
this.cr2filenames[cr].push(filename)
if (dateSeanceString != this.cr2date[cr]) {
info.push(
`${cr} meta QUANTIEME_SEANCE contains ${this.cr2date[cr]} instead of the expected ${dateSeanceString}`,
)
status = "needs-fixing"
}
} else {
if (status != "will-be-fixed") {
info.push(`${pattern} does not exist`)
status = "needs-fixing"
}
}
return { status: status, info: info }
}
findCandidates(filename: any): any {
const date = this.filename2date[filename]
if (!(date in this.date2crs)) return []
let candidates = []
for (const cr of this.date2crs[date]) {
if (this.cr2filenames[cr].length > 0)
candidates.push(
`${cr} is not a candidate because it is referenced by ${this.cr2filenames[cr]}`,
)
else candidates.push(`${cr} is a candidate`)
}
return candidates
}
postCheck(analysis: any): any {
for (const filename of Object.keys(analysis)) {
const result = analysis[filename]["reunion-00002"]
if (result == null || result["status"] != "needs-fixing") continue
const info = result["info"].join("")
if (
info.includes("meta QUANTIEME") ||
info.includes("idJo field is missing")
) {
result["info"] = result["info"].concat(this.findCandidates(filename))
}
}
}
fix(reunion: any): any {
if (!(reunion.uid in remapJo)) {
return null
}
const idJo = remapJo[reunion.uid]
const numSeanceJo = String(Number(idJo.substring(5)))
if (
reunion.identifiants.idJo == idJo &&
reunion.identifiants.numSeanceJo == numSeanceJo
)
return false
reunion.identifiants.idJo = idJo
reunion.identifiants.numSeanceJo = numSeanceJo
return true
}
}
......@@ -347,3 +347,69 @@ export const datasets: Datasets = {
},
],
}
const directory2schema: any = {
'.*Agenda_.*': 'reunion',
'.*Scrutins_.*': 'scrutin',
'.*Amendements_.*': 'amendement',
'.*Dossiers_Legislatifs_.*/documents': 'document',
'.*Dossiers_Legislatifs_.*/dossiers': 'dossier',
'.*acteurs_mandats_organes/organes': 'organe',
'.*acteurs_mandats_organes/acteurs': 'acteur',
'.*AMO30_tous_acteurs_tous_mandats_tous_organes_historique/organes': 'organe',
'.*AMO30_tous_acteurs_tous_mandats_tous_organes_historique/acteurs': 'acteur',
}
export function getDatasets(): any {
return [
'Agenda_XIV',
'Agenda_XV',
'Scrutins_XIV',
'Scrutins_XV',
'Amendements_XIV',
'Amendements_XV',
'Dossiers_Legislatifs_XV',
'Dossiers_Legislatifs_XIV',
'AMO30_tous_acteurs_tous_mandats_tous_organes_historique',
'acteurs_mandats_organes',
]
}
export function validDataset(dir: any): any {
for (const dataset of getDatasets()) {
if (dir.includes(dataset))
return true
}
return false
}
export function getSchemas(): any {
return Object.values(directory2schema)
}
export function datasetDirectorySchema(dataset: string): any {
let directories: any
if (dataset.includes('Dossiers_Legislatifs_')) {
directories = [
`${dataset}/documents`,
`${dataset}/dossiers`,
]
} else if(dataset.includes('acteurs_mandats_organes') ||
dataset.includes('AMO30_tous_acteurs_tous_mandats_tous_organes_historique')) {
directories = [
`${dataset}/organes`,
`${dataset}/acteurs`,
]
} else {
directories = [ dataset ]
}
let results = []
for (const directory of directories) {
for (const re of Object.keys(directory2schema)) {
if (new RegExp(re).exec(directory))
results.push([`${directory}/**/*.json`, directory2schema[re]])
}
}
return results
}
import glob from "glob"
import fs from "fs"
import fs from "fs-extra"
import path from "path"
export function* walkDir(
......@@ -28,6 +28,14 @@ export function load(path: string): any {
return JSON.parse(rawjson)
}
export function write(something: any, filename: string): any {
const content = JSON.stringify(something, null, 2)
fs.ensureDirSync(path.dirname(filename))
fs.writeFileSync(filename, content, {
encoding: "utf8",
})
}
export function getFiles(args: any): any {
let files: string[] = []
function _getFiles(fileOrPattern: any) {
......
import { execSync } from "child_process"
export function run(repositoryDir: string, args: string): string {
return execSync(`git ${args}`, { cwd: repositoryDir })
.toString()
.trim()
}
export function test(repositoryDir: string, args: string): boolean {
try {
execSync(`git ${args}`, {
cwd: repositoryDir,
stdio: ["ignore", "pipe", "pipe"],
})
return true
} catch (childProcess) {
if (childProcess.status != 0) return false
throw childProcess
}
}
export function commit(repositoryDir: string, message: string): boolean {
execSync("git add .", {
cwd: repositoryDir,
env: process.env,
encoding: "utf-8",
stdio: ["ignore", "ignore", "pipe"],
})
try {
execSync(`git commit -m "${message}"`, {
cwd: repositoryDir,
env: process.env,
encoding: "utf-8",
stdio: ["ignore", "pipe", "pipe"],
})
return true
} catch (childProcess) {
if (
childProcess.stdout === null ||
!/nothing to commit/.test(childProcess.stdout)
) {
console.error(childProcess.output)
throw childProcess
}
return false
}
}
......@@ -117,6 +117,6 @@
],
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "DossierLegislatif.json",
"$$target": "DossierLegislatif.json"
"$id": "Dossier.json",
"$$target": "Dossier.json"
}
import fs from "fs-extra"
import commandLineArgs from "command-line-args"
import commandLineUsage from "command-line-usage"
import { load, getFiles } from "../file_systems"
import { SchemaBugs } from "../../src/bugs"
import * as git from "../../src/git"
import {
validDataset,
getDatasets,
getSchemas,
datasetDirectorySchema,
} from "../../src/datasets"
function parseArgs(argv: string[]): any {
const optionsDefinitions = [
{
description: "pathname to the directory containing the dataset",
defaultOption: true,
name: "data",
type: String,
},
{
description:
"pathname to the directory containing the HTML comptes rendus de seance",
name: "cr",
type: String,
},
{
description: "pathname to the directory to commit",
name: "commit",
type: String,
},
{
description:
"pathname to the directory where the check outputs will be stored",
name: "output",
type: String,
},
{
description: `dataset (possible values must include exactly one of ${getDatasets()})`,
name: "dataset",
type: String,
},
{
description: `schema (possible values are ${getSchemas()})`,
name: "schema",
type: String,
},
{
description: "increase verbosity",
name: "verbose",
type: Boolean,
},
{
description: "check",
name: "check",
type: Boolean,
},
{
description: "fix",
name: "fix",
type: Boolean,
},
{
name: "help",
description: "Print this usage guide.",
},
]
const sections = [
{
header: "Cross check data consistency",
content: "Cross check data consistency",
},
{
header: "Options",
optionList: optionsDefinitions,
},
]
const options = commandLineArgs(optionsDefinitions, {
argv: argv,
})
if (options.dataset) {
if (!validDataset(options.dataset)) {
console.error(`--dataset ${options.dataset}`)
options.help = true
}
}
if (options.schema) {
if (!getSchemas().includes(options.schema)) {
console.error(`--schema ${options.schema}`)
options.help = true
}
}
if ("help" in options) {
const usage = commandLineUsage(sections)
console.warn(usage)
return null
}
return options
}
class BugsHelper {
options: any
bugs: any
constructor(options: any) {
this.options = options
this.bugs = new SchemaBugs(this.options)
}
async check(): Promise<number> {
const analysis: any = {}
this.bugs.preCheck()
for (const file of getFiles([this.options.data])) {
const content = load(file)
analysis[file] = this.bugs.check(content, file)
}
this.bugs.postCheck(analysis)
await this.show(analysis)
if (this.options.output) await this.store(analysis)
return 0
}
async show(analysis: any): Promise<void> {
for (const filename of Object.keys(analysis)) {
const bug = analysis[filename]
for (const name of Object.keys(bug)) {
const result = bug[name]
if (result != null) {
let log
if (result["status"] == "needs-fixing") {
log = console.error
} else if (this.options.verbose) {
log = console.log
}
if (log != undefined) {
log(`${name} finds ${filename} ${result["status"]}`)
log(result["info"].join("\n"))
}
}
}
}
}
async store(analysis: any): Promise<void> {
const name2reports: any = {}
for (const filename of Object.keys(analysis)) {
const bug = analysis[filename]
for (const name of Object.keys(bug)) {
if (!(name in name2reports)) name2reports[name] = {}
name2reports[name][filename] = bug[name]
}
}
for (const name of Object.keys(name2reports)) {
const reports = name2reports[name]
const lines: any = [`## ${name}`]
for (const filename of Object.keys(reports)) {
const report = reports[filename]
if (report == null || report["status"] != "needs-fixing") continue
lines.push(`* ${filename}`)
for (const info of report["info"]) lines.push(` * ${info}`)
}
const output = `${this.options.output}/${name}.md`
fs.writeFileSync(output, lines.join("\n"), {
encoding: "utf8",
})
}
}
async checkoutFix(): Promise<void> {
const c = this.options.commit
if (c == undefined) return
if (/tmp/.test(git.run(c, "branch"))) git.run(c, "branch --delete tmp")
git.run(c, "checkout --quiet -b tmp master")
if (/Merge bugfixes/.test(git.run(c, "log --oneline -1")))
git.run(c, "revert -m 1 HEAD")
git.run(c, "merge --no-ff -m 'Merge upstream' upstream")
if (/bugfixes/.test(git.run(c, "branch")))
git.run(c, "branch --delete bugfixes")
git.run(c, "checkout --quiet -b bugfixes")
}
async mergeFix(): Promise<void> {
const c = this.options.commit
if (c == undefined) return
git.run(c, "checkout --quiet tmp")
git.run(c, "merge --no-ff -m 'Merge bugfixes' bugfixes")
git.run(c, "branch --delete bugfixes")
if (!git.test(c, "diff --quiet tmp master")) {
git.run(c, "checkout --quiet master")
git.run(c, "merge --ff-only tmp")
}
git.run(c, "checkout --quiet master")
git.run(c, "branch -D tmp")
}
async fix(): Promise<number> {
await this.checkoutFix()
await this.bugs.fix(getFiles([this.options.data]))
await this.mergeFix()
return 0
}
async main(): Promise<number> {
if (this.options.check) return this.check()
else return this.fix()
}