bugs: CLI for fix & check

parent 1dc6348a
import fs from "fs-extra"
import commandLineArgs from "command-line-args"
import commandLineUsage from "command-line-usage"
import { load, getFiles } from "../file_systems"
import { SchemaBugs } from "../../src/bugs"
import * as git from "../../src/git"
import {
validDataset,
getDatasets,
getSchemas,
datasetDirectorySchema,
} from "../../src/datasets"
function parseArgs(argv: string[]): any {
const optionsDefinitions = [
{
description: "pathname to the directory containing the dataset",
defaultOption: true,
name: "data",
type: String,
},
{
description:
"pathname to the directory containing the HTML comptes rendus de seance",
name: "cr",
type: String,
},
{
description: "pathname to the directory to commit",
name: "commit",
type: String,
},
{
description:
"pathname to the directory where the check outputs will be stored",
name: "output",
type: String,
},
{
description: `dataset (possible values must include exactly one of ${getDatasets()})`,
name: "dataset",
type: String,
},
{
description: `schema (possible values are ${getSchemas()})`,
name: "schema",
type: String,
},
{
description: "increase verbosity",
name: "verbose",
type: Boolean,
},
{
description: "check",
name: "check",
type: Boolean,
},
{
description: "fix",
name: "fix",
type: Boolean,
},
{
name: "help",
description: "Print this usage guide.",
},
]
const sections = [
{
header: "Cross check data consistency",
content: "Cross check data consistency",
},
{
header: "Options",
optionList: optionsDefinitions,
},
]
const options = commandLineArgs(optionsDefinitions, {
argv: argv,
})
if (options.dataset) {
if (!validDataset(options.dataset)) {
console.error(`--dataset ${options.dataset}`)
options.help = true
}
}
if (options.schema) {
if (!getSchemas().includes(options.schema)) {
console.error(`--schema ${options.schema}`)
options.help = true
}
}
if ("help" in options) {
const usage = commandLineUsage(sections)
console.warn(usage)
return null
}
return options
}
class BugsHelper {
options: any
bugs: any
constructor(options: any) {
this.options = options
this.bugs = new SchemaBugs(this.options)
}
async check(): Promise<number> {
const analysis: any = {}
this.bugs.preCheck()
for (const file of getFiles([this.options.data])) {
const content = load(file)
analysis[file] = this.bugs.check(content, file)
}
this.bugs.postCheck(analysis)
await this.show(analysis)
if (this.options.output) await this.store(analysis)
return 0
}
async show(analysis: any): Promise<void> {
for (const filename of Object.keys(analysis)) {
const bug = analysis[filename]
for (const name of Object.keys(bug)) {
const result = bug[name]
if (result != null) {
let log
if (result["status"] == "needs-fixing") {
log = console.error
} else if (this.options.verbose) {
log = console.log
}
if (log != undefined) {
log(`${name} finds ${filename} ${result["status"]}`)
log(result["info"].join("\n"))
}
}
}
}
}
async store(analysis: any): Promise<void> {
const name2reports: any = {}
for (const filename of Object.keys(analysis)) {
const bug = analysis[filename]
for (const name of Object.keys(bug)) {
if (!(name in name2reports)) name2reports[name] = {}
name2reports[name][filename] = bug[name]
}
}
for (const name of Object.keys(name2reports)) {
const reports = name2reports[name]
const lines: any = [`## ${name}`]
for (const filename of Object.keys(reports)) {
const report = reports[filename]
if (report == null || report["status"] != "needs-fixing") continue
lines.push(`* ${filename}`)
for (const info of report["info"]) lines.push(` * ${info}`)
}
const output = `${this.options.output}/${name}.md`
fs.writeFileSync(output, lines.join("\n"), {
encoding: "utf8",
})
}
}
async checkoutFix(): Promise<void> {
const c = this.options.commit
if (c == undefined) return
if (/tmp/.test(git.run(c, "branch"))) git.run(c, "branch --delete tmp")
git.run(c, "checkout --quiet -b tmp master")
if (/Merge bugfixes/.test(git.run(c, "log --oneline -1")))
git.run(c, "revert -m 1 HEAD")
git.run(c, "merge --no-ff -m 'Merge upstream' upstream")
if (/bugfixes/.test(git.run(c, "branch")))
git.run(c, "branch --delete bugfixes")
git.run(c, "checkout --quiet -b bugfixes")
}
async mergeFix(): Promise<void> {
const c = this.options.commit
if (c == undefined) return
git.run(c, "checkout --quiet tmp")
git.run(c, "merge --no-ff -m 'Merge bugfixes' bugfixes")
git.run(c, "branch --delete bugfixes")
if (!git.test(c, "diff --quiet tmp master")) {
git.run(c, "checkout --quiet master")
git.run(c, "merge --ff-only tmp")
}
git.run(c, "checkout --quiet master")
git.run(c, "branch -D tmp")
}
async fix(): Promise<number> {
await this.checkoutFix()
await this.bugs.fix(getFiles([this.options.data]))
await this.mergeFix()
return 0
}
async main(): Promise<number> {
if (this.options.check) return this.check()
else return this.fix()
}
}
async function main(argv: any): Promise<number> {
const options = parseArgs(argv)
if (options === null) return 1
let directorySchema = []
if (options.schema) {
directorySchema = [[options.data, options.schema]]
} else if (options.dataset) {
directorySchema = datasetDirectorySchema(options.dataset)
} else {
console.error("either --schema or --dataset is required")
return 1
}
for (const d of directorySchema) {
options.schema = d[1]
options.data = d[0]
if (options.verbose)
console.log(
`working on files ${options.data} with schema ${options.schema}`,
)
const helper = new BugsHelper(options)
const status = await helper.main()
if (status != 0) return status
}
return 0
}
/* istanbul ignore if */
if (process.argv[1].endsWith("bugs_helper.ts"))
main(process.argv)
.then(status => {
process.exit(status)
})
.catch(error => {
console.error(error)
process.exit(1)
})
import fs from "fs-extra"
import temp from "temp"
let rewire = require("rewire")
let sinon = require("sinon")
import { assert } from "chai"
import * as git from "../src/git"
import { write } from "../src/file_systems"
let bugs_helper = rewire("../src/scripts/bugs_helper")
suite("bugs_helper")
test("#fixInRepository", async function() {
this.timeout(5000)
const BugsHelper = bugs_helper.__get__("BugsHelper")
const tmpDir = temp.mkdirSync("check")
git.run(tmpDir, "init")
git.run(tmpDir, "config commit.gpgsign false")
git.run(tmpDir, "config user.email 'tricoteuses@tricoteuses.fr'")
git.run(tmpDir, "config user.name 'Tricoteuses données Assemblée'")
const file = "RUANR5L15S2020IDS22089.json"
fs.copySync(`tests/check/${file}`, `${tmpDir}/${file}`)
git.run(tmpDir, "add .")
git.run(tmpDir, "commit -m 'initial commit'")
git.run(tmpDir, "checkout --quiet -b upstream master")
write("A", `${tmpDir}/upstreamfile`)
git.run(tmpDir, "add .")
git.run(tmpDir, "commit -m 'upstream commit'")
git.run(tmpDir, "checkout --quiet -b bugfixes master")
const h = new BugsHelper({
commit: tmpDir,
schema: "reunion",
data: `${tmpDir}/${file}`,
})
await h.fix()
assert.include(git.run(tmpDir, "log --oneline"), "Fix reunion")
assert.include(git.run(tmpDir, "log --oneline"), "upstream commit")
assert.notInclude(git.run(tmpDir, "branch"), "bugfixes")
assert.notInclude(git.run(tmpDir, "branch"), "Revert")
assert.include(git.run(tmpDir, "log --oneline -1"), "Merge bugfixes")
const hash = git.run(tmpDir, "rev-parse HEAD")
// nothing changed, do nothing
await h.fix()
assert.equal(hash, git.run(tmpDir, "rev-parse HEAD"))
// upstream changes, revert and re-apply fixes
git.run(tmpDir, "checkout --quiet upstream")
write("B", `${tmpDir}/upstreamfile`)
git.run(tmpDir, "add .")
git.run(tmpDir, "commit -m 'another upstream commit'")
git.run(tmpDir, "checkout --quiet master")
await h.fix()
assert.notEqual(hash, git.run(tmpDir, "rev-parse HEAD"))
assert.equal(
'Merge bugfixes\nFix reunion-00002\nMerge upstream\nRevert "Merge bugfixes"',
git.run(tmpDir, "log --date-order -4 --format=%s"),
)
})
test("#checkWillBeFixed", async function() {
const main = bugs_helper.__get__("main")
const log = sinon.stub(console, "log")
const warn = sinon.stub(console, "warn")
const error = sinon.stub(console, "error")
let status = await main([
"--verbose",
"--check",
"--schema=reunion",
"--cr=tests/check",
"--data=tests/check/RUANR5L15S2020IDS22089.json",
])
assert.equal(status, 0)
assert.deepEqual(warn.args, [])
warn.reset()
assert.deepEqual(error.args, [])
error.reset()
assert.include(log.args[2][0], "will-be-fixed")
log.reset()
log.restore()
warn.restore()
error.restore()
})
test("#checkDataset", async function() {
const main = bugs_helper.__get__("main")
const log = sinon.stub(console, "log")
const warn = sinon.stub(console, "warn")
const error = sinon.stub(console, "error")
let status = await main([
"--verbose",
"--check",
"--dataset=tests/check/Agenda_XV",
])
assert.deepEqual(warn.args, [])
warn.reset()
assert.deepEqual(error.args, [])
error.reset()
assert.equal(status, 0)
assert.include(log.args[2][0], "will-be-fixed")
log.reset()
log.restore()
warn.restore()
error.restore()
})
test("#checkCandidates", async function() {
const main = bugs_helper.__get__("main")
const log = sinon.stub(console, "log")
const warn = sinon.stub(console, "warn")
const error = sinon.stub(console, "error")
const tmpDir = temp.mkdirSync("check")
// good run
let status = await main([
"--verbose",
"--check",
`--output=${tmpDir}`,
"--schema=reunion",
"--cr=tests/check",
"--data=tests/check/RUANR5L15S2019IDS21272.json",
])
assert.equal(status, 0)
assert.deepEqual(warn.args, [])
warn.reset()
assert.deepEqual(error.args, [
[
"reunion-00002 finds tests/check/RUANR5L15S2019IDS21272.json needs-fixing",
],
["idJo field is missing\ntests/check/test_20190017.asp is a candidate"],
])
error.reset()
fs.existsSync(`${tmpDir}/reunion-00002.md`)
log.restore()
warn.restore()
error.restore()
})
test("#wrongArgument", async function() {
const main = bugs_helper.__get__("main")
const log = sinon.stub(console, "log")
const warn = sinon.stub(console, "warn")
const error = sinon.stub(console, "error")
for (const argument of ["schema", "dataset"]) {
const status = await main([
"--verbose",
"--check",
`--${argument}=INVALID`,
"--cr=tests/check",
"--data=tests/check/RUANR5L15S2020IDS22089.json",
])
assert.equal(status, 1)
assert.deepEqual(error.args, [[`--${argument} INVALID`]])
error.reset()
}
log.restore()
warn.restore()
error.restore()
})
test("#main", async function() {
const main = bugs_helper.__get__("main")
const log = sinon.stub(console, "log")
const warn = sinon.stub(console, "warn")
const error = sinon.stub(console, "error")
// good run
let status = await main([
"--verbose",
"--check",
"--schema=reunion",
"--cr=tests/check",
"--data=tests/check/RUANR5L15S2020IDS22089.json",
])
assert.equal(status, 0)
assert.deepEqual(warn.args, [])
warn.reset()
// missing mandatory --data option
assert.equal(await main([]), 1)
log.restore()
warn.restore()
error.restore()
})
{
"schemaVersion": "agenda-1.0",
"xsiType": "seance_type",
"uid": "RUANR5L15S2020IDS22089",
"timestampDebut": "2020-02-12T15:00:00.000+01:00",
"timestampFin": "2020-02-12T18:05:00.000+01:00",
"lieu": {
"code": "AN",
"libelleLong": "Assemblée nationale"
},
"cycleDeVie": {
"etat": "Confirmé",
"chrono": {
"creation": "2020-01-20T00:00:00.000+01:00"
}
},
"organeReuniRef": "PO717460",
"sessionRef": "SCR5A2020O1",
"ouverturePresse": true,
"odj": {
"pointsOdj": [
{
"xsiType": "podjSeanceConfPres_type",
"uid": "RUANR5L15S2020IDS22089PT38154",
"cycleDeVie": {
"etat": "Confirmé",
"chrono": {
"creation": "2020-01-20T00:00:00.000+01:00"
}
},
"objet": "Discussion, en deuxième lecture, de la proposition de loi, modifiée par le Sénat, visant à encourager la participation des citoyens aux premiers secours",
"procedure": "procédure de législation en commission-Article 107-1",
"dossiersLegislatifsRefs": [
"DLR5L15N37091"
],
"typePointOdj": "Discussion",
"comiteSecret": false,
"natureTravauxOdj": "ODJPR",
"dateConfPres": "2020-02-12T01:00:00.000+01:00"
},
{
"xsiType": "podjSeanceConfPres_type",
"uid": "RUANR5L15S2020IDS22089PT38155",
"cycleDeVie": {
"etat": "Confirmé",
"chrono": {
"creation": "2020-01-20T00:00:00.000+01:00"
}
},
"objet": "Discussion de la proposition de loi visant à encadrer l'exploitation commerciale de l'image d'enfants de moins de seize ans sur les plateformes en ligne",
"dossiersLegislatifsRefs": [
"DLR5L15N38385"
],
"typePointOdj": "Discussion",
"comiteSecret": false,
"natureTravauxOdj": "ODJPR"
}
]
},
"compteRenduRef": "CRSANR5L15S2020O1N141",
"identifiants": {
"numSeanceJo": "141",
"idJo": "20200141",
"quantieme": "Unique",
"dateSeance": "2020-02-12T01:00:00.000+01:00"
},
"captationVideo": true
}
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment