Merge remote-tracking branch 'origin/develop' into feature/issue-756_…

…airgap

.github/workflows/release.yml

@@ -3,6 +3,8 @@ name: "Generate Releases"
 on:
   release:
     types: [ published ]
+  schedule:
+    - cron: '0 12 1,15 * *'  # On the 1st and 15th of the month at noon
 # To test this workflow without creating a release, uncomment the following and add a branch name (making sure "push"
 #  is at the same indent level as "release":
   push:
@@ -182,7 +184,7 @@ jobs:
 
       - name: "Upload binaries to release"
         uses: svenstaro/upload-release-action@v2
-        if: github.event_name == 'release'
+        if: github.event_name == 'release' || github.event_name == 'schedule'
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           file: build/${{ steps.basefn.outputs.FILEPATH }}*

src/main/java/network/brightspots/rcv/BaseCvrReader.java

@@ -17,6 +17,7 @@
 package network.brightspots.rcv;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -40,16 +41,35 @@ abstract void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
       throws CastVoteRecord.CvrParseException, IOException;
 
   // Individual contests may have a different value than what the config allows.
-  public Integer getMaxRankingsAllowed(String contestId) {
-    return config.getMaxRankingsAllowed();
+  public boolean isRankingAllowed(int rank, String contestId) {
+    return config.isRankingAllowed(rank);
   }
 
   // Any reader-specific validations can override this function.
   public void runAdditionalValidations(List<CastVoteRecord> castVoteRecords)
-      throws CastVoteRecord.CvrParseException {}
+      throws CastVoteRecord.CvrParseException {
+    for (CastVoteRecord cvr : castVoteRecords) {
+      for (Pair<Integer, CandidatesAtRanking> ranking : cvr.candidateRankings) {
+        if (!this.config.isRankingAllowed(ranking.getKey())) {
+          throw new CastVoteRecord.CvrParseException();
+        }
+      }
+    }
+  }
+
+  // Some CVRs have a list of candidates in the file. Read that list and return it.
+  // This will be used in tandem with gatherUnknownCandidates, which only looks for candidates
+  // that have at least one vote.
+  public List<String> readCandidateListFromCvr(List<CastVoteRecord> castVoteRecords)
+      throws IOException {
+    return new ArrayList<>();
+  }
 
   // Gather candidate names from the CVR that are not in the config.
-  Map<String, Integer> gatherUnknownCandidates(List<CastVoteRecord> castVoteRecords) {
+  Map<String, Integer> gatherUnknownCandidates(
+      List<CastVoteRecord> castVoteRecords, boolean includeCandidatesWithZeroVotes) {
+    // First pass: gather all unrecognized candidates and their counts
+    // All CVR Readers have this implemented
     Map<String, Integer> unrecognizedCandidateCounts = new HashMap<>();
     for (CastVoteRecord cvr : castVoteRecords) {
       for (Pair<Integer, CandidatesAtRanking> ranking : cvr.candidateRankings) {
@@ -65,6 +85,32 @@ Map<String, Integer> gatherUnknownCandidates(List<CastVoteRecord> castVoteRecord
       }
     }
 
+    if (includeCandidatesWithZeroVotes) {
+      // Second pass: read the entire candidate list from the CVR,
+      // regardless of whether they have any votes.
+      // TODO -- once all readers have this implemented, we can skip the first pass entirely
+      // during auto-load candidates and just use readCandidateListFromCvr.
+      List<String> allCandidates = new ArrayList<>();
+      try {
+        allCandidates = readCandidateListFromCvr(castVoteRecords);
+      } catch (IOException e) {
+        // If we can't read the candidate list, we can't check for unrecognized candidates.
+        Logger.warning("IOException reading candidate list from CVR: %s", e.getMessage());
+      }
+
+      // Remove overvote and write-in label from candidate list, if they exist
+      allCandidates.remove(source.getOvervoteLabel());
+      allCandidates.remove(source.getUndeclaredWriteInLabel());
+
+      // Combine the lists
+      for (String candidateName : allCandidates) {
+        if (!unrecognizedCandidateCounts.containsKey(candidateName)
+            && config.getNameForCandidate(candidateName) == null) {
+          unrecognizedCandidateCounts.put(candidateName, 0);
+        }
+      }
+    }
+
     return unrecognizedCandidateCounts;
   }
 

src/main/java/network/brightspots/rcv/CastVoteRecord.java

@@ -66,23 +66,31 @@ class CastVoteRecord {
       String precinct,
       String precinctPortion,
       List<Pair<Integer, String>> rankings) {
+    this(contestId, tabulatorId, batchId, suppliedId, null, precinct, precinctPortion, rankings);
+  }
+
+  CastVoteRecord(
+          String contestId,
+          String tabulatorId,
+          String batchId,
+          String suppliedId,
+          String computedId,
+          String precinct,
+          String precinctPortion,
+          List<Pair<Integer, String>> rankings) {
     this.contestId = contestId;
     this.tabulatorId = tabulatorId;
     this.batchId = batchId;
-    this.computedId = null;
     this.suppliedId = suppliedId;
+    this.computedId = computedId;
     this.precinct = precinct;
     this.precinctPortion = precinctPortion;
     this.candidateRankings = new CandidateRankingsList(rankings);
   }
 
   CastVoteRecord(
       String computedId, String suppliedId, String precinct, List<Pair<Integer, String>> rankings) {
-    this.computedId = computedId;
-    this.suppliedId = suppliedId;
-    this.precinct = precinct;
-    this.precinctPortion = null;
-    this.candidateRankings = new CandidateRankingsList(rankings);
+    this(null, null, null, suppliedId, computedId, precinct, null, rankings);
   }
 
   String getContestId() {
@@ -115,10 +123,10 @@ void logRoundOutcome(
 
     StringBuilder logStringBuilder = new StringBuilder();
     logStringBuilder.append("[Round] ").append(round).append(" [CVR] ");
-    if (!isNullOrBlank(suppliedId)) {
-      logStringBuilder.append(suppliedId);
-    } else {
+    if (!isNullOrBlank(computedId)) {
       logStringBuilder.append(computedId);
+    } else {
+      logStringBuilder.append(suppliedId);
     }
     if (outcomeType == VoteOutcomeType.IGNORED) {
       logStringBuilder.append(" [was ignored] ");

src/main/java/network/brightspots/rcv/ClearBallotCvrReader.java

@@ -54,7 +54,7 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
         Logger.severe("No header row found in cast vote record file: %s", this.cvrPath);
         throw new CvrParseException();
       }
-      String[] headerData = firstRow.split(",");
+      String[] headerData = firstRow.split(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)");
       if (headerData.length < CvrColumnField.ChoicesBegin.ordinal()) {
         Logger.severe("No choice columns found in cast vote record file: %s", this.cvrPath);
         throw new CvrParseException();
@@ -82,13 +82,9 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
           choiceName = Tabulator.UNDECLARED_WRITE_IN_OUTPUT_LABEL;
         }
         Integer rank = Integer.parseInt(choiceFields[RcvChoiceHeaderField.RANK.ordinal()]);
-        if (rank > this.config.getMaxRankingsAllowed()) {
-          Logger.severe(
-              "Rank: %d exceeds max rankings allowed in config: %d",
-              rank, this.config.getMaxRankingsAllowed());
-          throw new CvrParseException();
+        if (this.config.isRankingAllowed(rank)) {
+          columnIndexToRanking.put(columnIndex, new Pair<>(rank, choiceName));
         }
-        columnIndexToRanking.put(columnIndex, new Pair<>(rank, choiceName));
       }
       // read all remaining rows and create CastVoteRecords for each one
       while (true) {

src/main/java/network/brightspots/rcv/ContestConfig.java

@@ -678,9 +678,9 @@ private void validateRules() {
       Logger.severe("Invalid winnerElectionMode!");
     }
 
-    if (getMaxRankingsAllowed() == null
+    if (getMaxRankingsAllowedAsString() == null
         || (getNumDeclaredCandidates() >= 1
-            && getMaxRankingsAllowed() < MIN_MAX_RANKINGS_ALLOWED)) {
+            && !isRankingAllowed(MIN_MAX_RANKINGS_ALLOWED))) {
       validationErrors.add(ValidationError.RULES_MAX_RANKINGS_ALLOWED_INVALID);
       Logger.severe(
           "maxRankingsAllowed must either be \"%s\" or an integer from %d to %d!",
@@ -988,11 +988,47 @@ private String getMaxRankingsAllowedRaw() {
     return rawConfig.rules.maxRankingsAllowed;
   }
 
-  Integer getMaxRankingsAllowed() {
-    return stringToIntWithOption(
-        getMaxRankingsAllowedRaw(),
-        MAX_RANKINGS_ALLOWED_NUM_CANDIDATES_OPTION,
-        getNumDeclaredCandidates());
+  // Because the max rank can be set to "max", there's no single value that makes sense
+  // to all callers of this function.
+  // Returning MAX_VALUE could cause integer overflows if
+  // the caller tries to do arithmetic with the result (e.g. convert to a max column),
+  // or if the caller is iterating for all values up until the max rank.
+  // Returning NumDeclaredCandidates is not valid in cases where this is used before
+  // candidates are declared.
+  // Instead of returning the max rank, only allow checking via this function,
+  // or returning the value as a string for audit logging.
+  boolean isRankingAllowed(int rank) {
+    if (rank <= 0) {
+      return false;
+    }
+
+    if (isMaxRankingsSetToMaximum()) {
+      return true;
+    }
+
+    return rank <= Integer.parseInt(getMaxRankingsAllowedRaw());
+  }
+
+  // There are times when it is necessary to grab the max ranking, for example,
+  // when iterating up until the max ranking, or when reading inputs where a
+  // "blank" indicates an undeclared write-in.
+  // Force the caller of this function to check isMaxRankingsSetToMaximum first,
+  // so we know they've handled that special case.
+  Integer getMaxRankingsAllowedWhenNotSetToMaximum() {
+    if (isMaxRankingsSetToMaximum()) {
+      throw new RuntimeException(
+              "Do not call this function without first checking isMaxRankingsSetToMaximum!");
+    }
+
+    return Integer.parseInt(getMaxRankingsAllowedRaw());
+  }
+
+  boolean isMaxRankingsSetToMaximum() {
+    return getMaxRankingsAllowedRaw().equals(MAX_RANKINGS_ALLOWED_NUM_CANDIDATES_OPTION);
+  }
+
+  String getMaxRankingsAllowedAsString() {
+    return getMaxRankingsAllowedRaw();
   }
 
   boolean isBatchEliminationEnabled() {

src/main/java/network/brightspots/rcv/CsvCvrReader.java

@@ -45,6 +45,25 @@ public String readerName() {
     return "generic CSV";
   }
 
+  @Override
+  public List<String> readCandidateListFromCvr(List<CastVoteRecord> castVoteRecords)
+      throws IOException {
+    try (FileInputStream inputStream = new FileInputStream(Path.of(cvrPath).toFile())) {
+      CSVParser parser =
+              CSVParser.parse(
+                      inputStream,
+                      Charset.defaultCharset(),
+                      CSVFormat.Builder.create().setHeader().build());
+      List<String> rawCandidateNames = parser.getHeaderNames();
+      // Split rawCandidateNames from firstVoteColumnIndex to the end
+      return new ArrayList<>(rawCandidateNames.subList(
+            firstVoteColumnIndex, rawCandidateNames.size()));
+    } catch (IOException exception) {
+      Logger.severe("Error parsing cast vote record:\n%s", exception);
+      throw exception;
+    }
+  }
+
   // parse CVR CSV file into CastVoteRecord objects and add them to the input List<CastVoteRecord>
   @Override
   void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
@@ -60,7 +79,9 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
 
       parser.stream().skip(firstVoteRowIndex);
 
+      int index = 0;
       for (CSVRecord csvRecord : parser) {
+        index++;
         ArrayList<Pair<Integer, String>> rankings = new ArrayList<>();
         for (int col = firstVoteColumnIndex; col < csvRecord.size(); col++) {
           String rankAsString = csvRecord.get(col);
@@ -86,7 +107,7 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
 
         // create the new CastVoteRecord
         CastVoteRecord newCvr =
-            new CastVoteRecord(source.getContestId(), "no supplied ID", "no precinct", rankings);
+            new CastVoteRecord(Integer.toString(index), "no supplied ID", "no precinct", rankings);
         castVoteRecords.add(newCvr);
       }
     } catch (IOException exception) {

src/main/java/network/brightspots/rcv/DominionCvrReader.java

@@ -30,6 +30,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import javafx.util.Pair;
 import network.brightspots.rcv.CastVoteRecord.CvrParseException;
 
@@ -171,12 +173,13 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords) throws CvrParseEx
   @Override
   public void runAdditionalValidations(List<CastVoteRecord> castVoteRecords)
       throws CastVoteRecord.CvrParseException {
+    super.runAdditionalValidations(castVoteRecords);
     validateNamesAreInContest(castVoteRecords);
   }
 
   @Override
-  public Integer getMaxRankingsAllowed(String contestId) {
-    return contests.get(contestId).getMaxRanks();
+  public boolean isRankingAllowed(int rank, String contestId) {
+    return rank > 0 && rank <= contests.get(contestId).getMaxRanks();
   }
 
   private void validateNamesAreInContest(List<CastVoteRecord> castVoteRecords)
@@ -270,6 +273,10 @@ private int parseCvrFile(
       String batchId = session.get("BatchId").toString();
       Integer recordId = (Integer) session.get("RecordId");
       String suppliedId = recordId.toString();
+      String computedId = Stream.of(tabulatorId, batchId, Integer.toString(recordId))
+              .filter(s -> s != null && !s.isEmpty())
+              .collect(Collectors.joining("|"));
+
       // filter out records which are not current and replace them with adjudicated ones
       HashMap adjudicatedData = (HashMap) session.get("Original");
       boolean isCurrent = (boolean) adjudicatedData.get("IsCurrent");
@@ -348,8 +355,8 @@ private int parseCvrFile(
           }
           // create the new cvr
           CastVoteRecord newCvr =
-              new CastVoteRecord(
-                  contestId, tabulatorId, batchId, suppliedId, precinct, precinctPortion, rankings);
+              new CastVoteRecord(contestId, tabulatorId, batchId, suppliedId,
+                      computedId, precinct, precinctPortion, rankings);
           castVoteRecords.add(newCvr);
         }
       }