Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for multi-file Dominion format #569

Merged
merged 11 commits into from
Jul 17, 2022
Merged

Add support for multi-file Dominion format #569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
d4ac2db
Merge pull request #541 from BrightSpots/develop
HEdingfield Dec 13, 2020
e94e377
Add support for multi-file Dominion format
catrope May 18, 2021
5b5aa88
Make multi-file Dominion progress logging quieter
catrope May 18, 2021
b5cd305
Correctly parse Dominion CVRs with multiple "Cards" in one "Session"
catrope May 20, 2021
90d0ae2
Switch names of the parseCvrFile and gatherCvrsForContest methods
catrope Mar 10, 2022
64b76e5
Don't initialize two variables on the same line
catrope Mar 10, 2022
dbab99c
Add tests for the multi-file Dominion format
catrope Mar 10, 2022
6ee20ac
remove test files before merging with new submodule
moldover Jul 11, 2022
523e2d6
Merge branch 'develop' into dominion-multi-file
moldover Jul 11, 2022
340c55b
add dominion multi file tests into new repo location
moldover Jul 11, 2022
d9ed6e5
Fix exception issue found by SpotBugs
moldover Jul 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 103 additions & 61 deletions src/main/java/network/brightspots/rcv/DominionCvrReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package network.brightspots.rcv;

import java.io.File;
import java.io.FileNotFoundException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
Expand All @@ -38,6 +39,7 @@ class DominionCvrReader {
private static final String CANDIDATE_MANIFEST = "CandidateManifest.json";
private static final String CONTEST_MANIFEST = "ContestManifest.json";
private static final String CVR_EXPORT = "CvrExport.json";
private static final String CVR_EXPORT_PATTERN = "CvrExport_%d.json";
private final ContestConfig config;
private final String manifestFolder;
private final String undeclaredWriteInLabel;
Expand Down Expand Up @@ -161,9 +163,8 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords, String contestId)
Logger.severe("No candidate data found!");
throw new CvrParseException();
}
// parse the cvr
Path cvrPath = Paths.get(manifestFolder, CVR_EXPORT);
parseCvrFile(cvrPath.toString(), castVoteRecords, contestId);
// parse the cvr file(s)
gatherCvrsForContest(castVoteRecords, contestId);
if (castVoteRecords.isEmpty()) {
Logger.severe("No cast vote record data found!");
throw new CvrParseException();
Expand All @@ -173,9 +174,8 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords, String contestId)
}
}

// parse the given file into a List of CastVoteRecords for tabulation
private void parseCvrFile(
String filePath, List<CastVoteRecord> castVoteRecords, String contestIdToLoad) {
// parse the CVR file or files into a List of CastVoteRecords for tabulation
private void gatherCvrsForContest(List<CastVoteRecord> castVoteRecords, String contestIdToLoad) {
// build a lookup map for candidates codes to optimize Cvr parsing
Map<String, Set<String>> contestIdToCandidateCodes = new HashMap<>();
for (Candidate candidate : this.candidates) {
Expand All @@ -190,58 +190,100 @@ private void parseCvrFile(
}

try {
HashMap json = JsonParser.readFromFile(filePath, HashMap.class);
// top-level "Sessions" object contains a lists of Cvr objects from different tabulators
ArrayList sessions = (ArrayList) json.get("Sessions");
// for each Cvr object extract various fields
for (Object sessionObject : sessions) {
HashMap session = (HashMap) sessionObject;
// extract various ids
String tabulatorId = session.get("TabulatorId").toString();
String batchId = session.get("BatchId").toString();
Integer recordId = (Integer) session.get("RecordId");
String suppliedId = recordId.toString();
// filter out records which are not current and replace them with adjudicated ones
HashMap adjudicatedData = (HashMap) session.get("Original");
boolean isCurrent = (boolean) adjudicatedData.get("IsCurrent");
if (!isCurrent) {
if (session.containsKey("Modified")) {
adjudicatedData = (HashMap) session.get("Modified");
} else {
Logger.warning(
"CVR has no adjudicated rankings, skipping: "
+ "Tabulator ID: %s Batch ID: %s Record ID: %d",
tabulatorId, batchId, recordId);
continue;
Path singleCvrPath = Paths.get(manifestFolder, CVR_EXPORT);
Path firstCvrPath = Paths.get(manifestFolder, String.format(CVR_EXPORT_PATTERN, 1));
if (singleCvrPath.toFile().exists()) {
HashMap json = JsonParser.readFromFile(singleCvrPath.toString(), HashMap.class);
parseCvrFile(json, castVoteRecords, contestIdToLoad, contestIdToCandidateCodes);
} else if (firstCvrPath.toFile().exists()) {
int recordsParsed = 0;
int recordsParsedAtLastlog = 0;
int cvrSequence = 1;
Path cvrPath = Paths.get(manifestFolder, String.format(CVR_EXPORT_PATTERN, cvrSequence));
while (cvrPath.toFile().exists()) {
HashMap json = JsonParser.readFromFile(cvrPath.toString(), HashMap.class);
recordsParsed += parseCvrFile(json, castVoteRecords, contestIdToLoad,
contestIdToCandidateCodes);
if (recordsParsed - recordsParsedAtLastlog > 50000) {
moldover marked this conversation as resolved.
Show resolved Hide resolved
Logger.info("Parsed %d records from %d files", recordsParsed, cvrSequence);
recordsParsedAtLastlog = recordsParsed;
}
cvrSequence++;
cvrPath = Paths.get(manifestFolder, String.format(CVR_EXPORT_PATTERN, cvrSequence));
}
// validate precinct (may not exist for older data sets)
Integer precinctId = (Integer) adjudicatedData.get("PrecinctId");
if (precinctId != null
&& (this.precincts == null || !this.precincts.containsKey(precinctId))) {
Logger.severe("Precinct ID \"%d\" from CVR not found in manifest data!", precinctId);
throw new CvrParseException();
}
String precinct = this.precincts != null ? this.precincts.get(precinctId) : null;
// validate precinct portion
Integer precinctPortionId = (Integer) adjudicatedData.get("PrecinctPortionId");
if (precinctPortionId != null && !this.precinctPortions.containsKey(precinctPortionId)) {
Logger.severe(
"Precinct portion ID \"%d\" from CVR not found in manifest data!", precinctPortionId);
throw new CvrParseException();
}
String precinctPortion = this.precinctPortions.get(precinctPortionId);
String ballotTypeId = adjudicatedData.get("BallotTypeId").toString();
} else {
throw new FileNotFoundException(String.format(
"Error parsing cast vote record: neither %s nor %s exists",
singleCvrPath.toString(), firstCvrPath.toString()));
}
} catch (FileNotFoundException | CvrParseException exception) {
Logger.severe("Error parsing cast vote record:\n%s", exception);
castVoteRecords.clear();
}
}

ArrayList contests;
// sometimes there is a "Cards" object at this level
if (adjudicatedData.containsKey("Cards")) {
ArrayList cardsList = (ArrayList) adjudicatedData.get("Cards");
HashMap cardsObject = (HashMap) cardsList.get(0);
contests = (ArrayList) cardsObject.get("Contests");
private int parseCvrFile(
HashMap json, List<CastVoteRecord> castVoteRecords, String contestIdToLoad,
Map<String, Set<String>> contestIdToCandidateCodes)
throws CvrParseException {
// top-level "Sessions" object contains a lists of Cvr objects from different tabulators
ArrayList sessions = (ArrayList) json.get("Sessions");
int recordsParsed = 0;
// for each Cvr object extract various fields
for (Object sessionObject : sessions) {
HashMap session = (HashMap) sessionObject;
// extract various ids
String tabulatorId = session.get("TabulatorId").toString();
String batchId = session.get("BatchId").toString();
Integer recordId = (Integer) session.get("RecordId");
String suppliedId = recordId.toString();
// filter out records which are not current and replace them with adjudicated ones
HashMap adjudicatedData = (HashMap) session.get("Original");
boolean isCurrent = (boolean) adjudicatedData.get("IsCurrent");
if (!isCurrent) {
if (session.containsKey("Modified")) {
adjudicatedData = (HashMap) session.get("Modified");
} else {
contests = (ArrayList) adjudicatedData.get("Contests");
Logger.warning(
"CVR has no adjudicated rankings, skipping: "
+ "Tabulator ID: %s Batch ID: %s Record ID: %d",
tabulatorId, batchId, recordId);
continue;
}
}
// validate precinct (may not exist for older data sets)
Integer precinctId = (Integer) adjudicatedData.get("PrecinctId");
if (precinctId != null
&& (this.precincts == null || !this.precincts.containsKey(precinctId))) {
Logger.severe("Precinct ID \"%d\" from CVR not found in manifest data!",
precinctId);
throw new CvrParseException();
}
String precinct = this.precincts != null ? this.precincts.get(precinctId) : null;
// validate precinct portion
Integer precinctPortionId = (Integer) adjudicatedData.get("PrecinctPortionId");
if (precinctPortionId != null && !this.precinctPortions.containsKey(precinctPortionId)) {
Logger.severe(
"Precinct portion ID \"%d\" from CVR not found in manifest data!",
precinctPortionId);
throw new CvrParseException();
}
String precinctPortion = this.precinctPortions.get(precinctPortionId);
String ballotTypeId = adjudicatedData.get("BallotTypeId").toString();

ArrayList cardsList;
// sometimes there is a "Cards" object at this level
if (adjudicatedData.containsKey("Cards")) {
cardsList = (ArrayList<Object>) adjudicatedData.get("Cards");
} else {
ArrayList<HashMap> oneCardList = new ArrayList<HashMap>(1);
oneCardList.add(adjudicatedData);
cardsList = (ArrayList) oneCardList;
}

for (Object cardObject : cardsList) {
HashMap card = (HashMap) cardObject;
ArrayList contests = (ArrayList) card.get("Contests");

// each contest object is a cvr
for (Object contestObject : contests) {
Expand Down Expand Up @@ -272,7 +314,8 @@ private void parseCvrFile(
Set<String> candidates = contestIdToCandidateCodes.get(contestId);
if (!candidates.contains(candidateCode)) {
Logger.severe(
"Candidate code '%s' is not valid for contest '%d'!", candidateCode, contestId);
"Candidate code '%s' is not valid for contest '%d'!", candidateCode,
contestId);
throw new CvrParseException();
}
// We also need to throw an error if this candidate doesn't appear in the tabulator's
Expand Down Expand Up @@ -300,15 +343,14 @@ private void parseCvrFile(
rankings);
castVoteRecords.add(newCvr);
}
// provide some user feedback on the Cvr count
if (castVoteRecords.size() % 50000 == 0) {
Logger.info("Parsed %d cast vote records.", castVoteRecords.size());
}
}
} catch (Exception exception) {
Logger.severe("Error parsing cast vote record:\n%s", exception);
castVoteRecords.clear();
// provide some user feedback on the Cvr count
recordsParsed++;
if (recordsParsed > 0 && recordsParsed % 50000 == 0) {
Logger.info("Parsed %d cast vote records.", recordsParsed);
}
}
return recordsParsed;
}

// Candidate data from a Dominion candidate manifest Json
Expand Down
6 changes: 6 additions & 0 deletions src/test/java/network/brightspots/rcv/TabulatorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,12 @@ void testDominionNoPrecinctData() {
runTabulationTest("dominion_no_precinct_data");
}

@Test
@DisplayName("multi-cvr file dominion test")
void multiFileDominionTest() {
runTabulationTest("dominion_multi_file");
}

@Test
@DisplayName("test invalid params in config file")
void invalidParamsTest() {
Expand Down
2 changes: 1 addition & 1 deletion src/test/resources/network/brightspots/rcv/test_data
Open in desktop
Submodule test_data updated from 05cef1 to 99486a