Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

CLDR-15850 Collect type of vote: direct, auto/manual import, bulk upload #2777

Merged
merged 5 commits into from
Mar 14, 2023
Merged

CLDR-15850 Collect type of vote: direct, auto/manual import, bulk upload #2777

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8b15f98
CLDR-15850 Collect type of vote: direct, auto/manual import, bulk upload
btangmu Mar 13, 2023
6127f22
CLDR-15850 Collect type of vote: direct, auto/manual import, bulk upload
btangmu Mar 13, 2023
3dd740c
CLDR-15850 Collect type of vote: direct, auto/manual import, bulk upload
btangmu Mar 14, 2023
7bd807a
CLDR-15850 Collect type of vote: direct, auto/manual import, bulk upload
btangmu Mar 14, 2023
93db439
CLDR-15850 Collect type of vote: direct, auto/manual import, bulk upload
btangmu Mar 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 7 additions & 9 deletions tools/cldr-apps/src/main/java/org/unicode/cldr/web/BallotBox.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import org.json.JSONObject;
import org.unicode.cldr.util.VoteResolver;
import org.unicode.cldr.util.VoteType;
import org.unicode.cldr.web.UserRegistry.User;

/**
Expand All @@ -18,6 +19,7 @@
* concrete type.
*/
public interface BallotBox<T> {

/**
* This is thrown when an XPath isn't valid within this locale.
* @author srl
Expand Down Expand Up @@ -59,14 +61,6 @@ public VoteNotAcceptedException(ErrorCode r, String string, JSONObject err_data)
}
}

/**
* As a special signal, when the "withVote" parameter of voteForValue is VOTE_IS_AUTO_IMPORTED,
* the ordinary vote count applies (as though the parameter were null), and some behavior is
* inhibited, such as auto-creation of forum posts in response to voting.
* It is negative to prevent confusion with valid positive numbers.
*/
public static final Integer VOTE_IS_AUTO_IMPORTED = -1;

/**
* Record a vote for an item. Will (eventually) throw a number of
* exceptions.
Expand All @@ -77,14 +71,18 @@ public VoteNotAcceptedException(ErrorCode r, String string, JSONObject err_data)
* dpath of item
* @param value
* new string value to vote for, or null for "unvote"
* @return the full xpath of the user's vote, or null if not applicable.
* @throws InvalidXPathException
* @throws VoteNotAcceptedException
*/
public void voteForValue(T user, String distinguishingXpath, String value, Integer withVote) throws InvalidXPathException, VoteNotAcceptedException;

public void voteForValue(T user, String distinguishingXpath, String value) throws InvalidXPathException, VoteNotAcceptedException;

public void voteForValueWithType(T user, String distinguishingXpath, String value, VoteType voteType) throws VoteNotAcceptedException, InvalidXPathException;

public void voteForValueWithType(T user, String distinguishingXpath, String value, Integer withVote, VoteType voteType) throws InvalidXPathException,
VoteNotAcceptedException;

/**
* Return a vote for a value, as a string
*
Expand Down
94 changes: 36 additions & 58 deletions tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,11 @@ public int getSize(CLDRLocale locale, String xpath) {

}

/**
* Names of some columns in DBUtils.Table.VOTE_VALUE
*/
private static final String VOTE_OVERRIDE = "vote_override";
private static final String VOTE_TYPE = "vote_type";

private class DataBackedSource extends DelegateXMLSource {
PerLocaleData ballotBox;
Expand Down Expand Up @@ -991,13 +995,23 @@ public void revoteFor(User user, String distinguishingXpath) throws BallotBox.In
voteForValue(user, distinguishingXpath, oldValue);
}

@Override
public void voteForValue(User user, String distinguishingXpath, String value, Integer withVote) throws InvalidXPathException, VoteNotAcceptedException {
voteForValueWithType(user, distinguishingXpath, value, withVote, VoteType.DIRECT);
}

@Override
public void voteForValue(User user, String distinguishingXpath, String value) throws InvalidXPathException, VoteNotAcceptedException {
voteForValue(user, distinguishingXpath, value, null);
voteForValueWithType(user, distinguishingXpath, value, null, VoteType.DIRECT);
}

@Override
public synchronized void voteForValue(User user, String distinguishingXpath, String value, Integer withVote) throws BallotBox.InvalidXPathException,
public void voteForValueWithType(User user, String distinguishingXpath, String value, VoteType voteType) throws VoteNotAcceptedException, InvalidXPathException {
voteForValueWithType(user, distinguishingXpath, value, null, voteType);
}

@Override
public synchronized void voteForValueWithType(User user, String distinguishingXpath, String value, Integer withVote, VoteType voteType) throws BallotBox.InvalidXPathException,
BallotBox.VoteNotAcceptedException {
makeSureInPathsForFile(distinguishingXpath, user, value);
value = processValue(distinguishingXpath, value);
Expand All @@ -1011,11 +1025,7 @@ public synchronized void voteForValue(User user, String distinguishingXpath, Str
}

int xpathId = sm.xpt.getByXpath(distinguishingXpath);
boolean voteIsAutoImported = false;
if (VOTE_IS_AUTO_IMPORTED.equals(withVote)) {
withVote = null;
voteIsAutoImported = true;
} else if (withVote != null) {
if (withVote != null) {
Level level = user.getLevel();
if (withVote == level.getVotes(user.getOrganization())) {
withVote = null; // not an override
Expand All @@ -1042,7 +1052,7 @@ public synchronized void voteForValue(User user, String distinguishingXpath, Str
String oldVal = xmlsource.getValueAtDPath(distinguishingXpath);

if (!readonly) {
saveVoteToDb(user, distinguishingXpath, value, withVote, xpathId, voteIsAutoImported);
saveVoteToDb(user, distinguishingXpath, value, withVote, xpathId, voteType);
} else {
readonly();
}
Expand Down Expand Up @@ -1132,16 +1142,14 @@ private DisplayAndInputProcessor getProcessor() {
* @param xpathId
*/
private void saveVoteToDb(final User user, final String distinguishingXpath, final String value,
final Integer withVote, final int xpathId, boolean voteIsAutoImported) {
final Integer withVote, final int xpathId, VoteType voteType) {
boolean didClearFlag = false;
makeSource(false);
ElapsedTimer et = !SurveyLog.DEBUG ? null : new ElapsedTimer("{0} Recording PLD for " + locale + " "
+ distinguishingXpath + " : " + user + " voting for '" + value);
Connection conn = null;
PreparedStatement saveOld = null; // save off old value
PreparedStatement ps = null; // all for mysql, or 1st step for
// derby
PreparedStatement ps2 = null; // 2nd step for derby
PreparedStatement ps = null;
final boolean wasFlagged = getFlag(locale, xpathId); // do this outside of the txn..
int submitter = user.id;
try {
Expand All @@ -1152,13 +1160,8 @@ private void saveVoteToDb(final User user, final String distinguishingXpath, fin
// #1 - save the "VOTE_VALUE_ALT" ( possible proposal) value.
if (DBUtils.db_Mysql) {
add0 = "IGNORE";
// add1="ON DUPLICATE KEY IGNORE";
} else {
add2 = "and not exists (select * from " + DBUtils.Table.VOTE_VALUE_ALT
+ " where " + DBUtils.Table.VOTE_VALUE_ALT + ".locale="
+ DBUtils.Table.VOTE_VALUE + ".locale and "
+ DBUtils.Table.VOTE_VALUE_ALT + ".xpath=" + DBUtils.Table.VOTE_VALUE + ".xpath and "
+ DBUtils.Table.VOTE_VALUE_ALT + ".value=" + DBUtils.Table.VOTE_VALUE + ".value )";
throw new RuntimeException("Unexpected db type, expected " + DBUtils.db_Mysql);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not needed. As of #2778, DBUtils.db_Mysql is public static final boolean db_Mysql = true;. I'm just going to be deleting this line soon.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good!

}
String sql = "insert " + add0 + " into " + DBUtils.Table.VOTE_VALUE_ALT + " " + add1
+ " select " + DBUtils.Table.VOTE_VALUE + ".locale,"
Expand All @@ -1169,36 +1172,17 @@ private void saveVoteToDb(final User user, final String distinguishingXpath, fin
saveOld.executeUpdate();

// #2 - save the actual vote.
if (DBUtils.db_Mysql) { // use 'on duplicate key' syntax
ps = DBUtils.prepareForwardReadOnly(conn, "INSERT INTO " + DBUtils.Table.VOTE_VALUE
+ " (locale,xpath,submitter,value,last_mod," + VOTE_OVERRIDE + ") values (?,?,?,?,CURRENT_TIMESTAMP,?) "
+ "ON DUPLICATE KEY UPDATE locale=?,xpath=?,submitter=?,value=?,last_mod=CURRENT_TIMESTAMP," + VOTE_OVERRIDE + "=?");
int colNum = 6;
ps = DBUtils.prepareForwardReadOnly(conn, "INSERT INTO " + DBUtils.Table.VOTE_VALUE
+ " (locale,xpath,submitter,value,last_mod," + VOTE_OVERRIDE + "," + VOTE_TYPE + ") values (?,?,?,?,CURRENT_TIMESTAMP,?,?) "
+ "ON DUPLICATE KEY UPDATE locale=?,xpath=?,submitter=?,value=?,last_mod=CURRENT_TIMESTAMP," + VOTE_OVERRIDE + "=?," + VOTE_TYPE + "=?");
int colNum = 1;
for (int repeat = 1; repeat <= 2; repeat++) {
ps.setString(colNum++, locale.getBaseName());
ps.setInt(colNum++, xpathId);
ps.setInt(colNum++, submitter);
DBUtils.setStringUTF8(ps, colNum++, value);
DBUtils.setInteger(ps, colNum++, withVote);
} else { // derby
ps2 = DBUtils.prepareForwardReadOnly(conn, "DELETE FROM " + DBUtils.Table.VOTE_VALUE
+ " where locale=? and xpath=? and submitter=? ");
ps = DBUtils.prepareForwardReadOnly(conn, "INSERT INTO " + DBUtils.Table.VOTE_VALUE
+ " (locale,xpath,submitter,value,last_mod," + VOTE_OVERRIDE + ") VALUES (?,?,?,?,CURRENT_TIMESTAMP,?) ");
int colNum = 1;
ps2.setString(colNum++, locale.getBaseName());
ps2.setInt(colNum++, xpathId);
ps2.setInt(colNum++, submitter);
// NB: no "VOTE_OVERRIDE" column on delete.
}

int colNum = 1;
ps.setString(colNum++, locale.getBaseName());
ps.setInt(colNum++, xpathId);
ps.setInt(colNum++, submitter);
DBUtils.setStringUTF8(ps, colNum++, value);
DBUtils.setInteger(ps, colNum++, withVote);
if (ps2 != null) {
ps2.executeUpdate();
DBUtils.setInteger(ps, colNum++, voteType.id());
}
ps.executeUpdate();

Expand All @@ -1209,14 +1193,16 @@ private void saveVoteToDb(final User user, final String distinguishingXpath, fin
conn.commit();
} catch (SQLException e) {
SurveyLog.logException(logger, e, "Exception in saveVoteToDb");
SurveyMain.busted("Could not vote for value in locale locale " + locale, e);
SurveyMain.busted("Could not vote for value in locale " + locale, e);
throw new InternalError("Could not load locale " + locale + " : " + DBUtils.unchainSqlException(e));
} finally {
DBUtils.close(saveOld, ps, ps2, conn);
DBUtils.close(saveOld, ps, conn);
}
SurveyLog.debug(et);

if (sm.fora != null && !voteIsAutoImported) {
// Voting can trigger adding a forum post (agree/decline) and/or closing a forum thread.
// AUTO_IMPORT and MANUAL_IMPORT votes are excluded; DIRECT and BULK_UPLOAD are not excluded.
if (sm.fora != null && (voteType != VoteType.AUTO_IMPORT && voteType != VoteType.MANUAL_IMPORT)) {
sm.fora.doForumAfterVote(locale, user, distinguishingXpath, xpathId, value, didClearFlag);
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The exclusion of MANUAL_IMPORT is new -- I'm not sure whether such forum posts have been made, but they would be very strange given that such votes are done in the name of an anonymous "user", and so the forum posts would also be from anonymous users (whose votes have zero weight)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I checked the db -- there has been exactly one forum post by an anonymous user:

mysql> select id,poster,loc,xpath,last_time,version,type,is_open from cldr_forum_posts where poster >= 2055 and poster <= 2074 limit 10;
+-------+--------+------+--------+---------------------+---------+------+---------+
| id    | poster | loc  | xpath  | last_time           | version | type | is_open |
+-------+--------+------+--------+---------------------+---------+------+---------+
| 53706 |   2055 | ps   | 687913 | 2020-07-06 04:29:12 | 38      |    3 |       0 |
+-------+--------+------+--------+---------------------+---------+------+---------+
1 row in set (0.01 sec)

It's an "Agree" post (type 3).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://st.unicode.org/cldr-apps/v#/ps/EnergyPower/376496946a9c5f0b

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know how an anonymous user would post. Almost sounds like an error?

}
}
Expand Down Expand Up @@ -1748,24 +1734,16 @@ private synchronized void setupDB() {
Statement s = null;
try (Connection conn = DBUtils.getInstance().getDBConnection()) {
if (!DBUtils.hasTable(DBUtils.Table.VOTE_VALUE.toString())) {
/*
* CREATE TABLE cldr_votevalue ( locale VARCHAR(20), xpath INT
* NOT NULL, submitter INT NOT NULL, value BLOB );
*
* CREATE UNIQUE INDEX cldr_votevalue_unique ON cldr_votevalue
* (locale,xpath,submitter);
*/
s = conn.createStatement();

sql = "create table " + DBUtils.Table.VOTE_VALUE + "( "
sql = "CREATE TABLE " + DBUtils.Table.VOTE_VALUE + "( "
+ "locale VARCHAR(20), "
+ "xpath INT NOT NULL, "
+ "submitter INT NOT NULL, " + "value " + DBUtils.DB_SQL_UNICODE + ", "
+ DBUtils.DB_SQL_LAST_MOD + ", "
+ VOTE_OVERRIDE + " INT DEFAULT NULL, "
+ " PRIMARY KEY (locale,submitter,xpath) " +

" )";
+ VOTE_TYPE + " TINYINT NOT NULL, "
+ "PRIMARY KEY (locale,submitter,xpath) " +
")";
// logger.info(sql);
s.execute(sql);

Expand Down
20 changes: 3 additions & 17 deletions tools/cldr-apps/src/main/java/org/unicode/cldr/web/SurveyAjax.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,25 +37,11 @@
import org.unicode.cldr.test.DisplayAndInputProcessor;
import org.unicode.cldr.test.SubmissionLocales;
import org.unicode.cldr.test.TestCache;
import org.unicode.cldr.util.CLDRConfig;
import org.unicode.cldr.util.CLDRConfigImpl;
import org.unicode.cldr.util.CLDRFile;
import org.unicode.cldr.util.*;
import org.unicode.cldr.util.CLDRInfo.CandidateInfo;
import org.unicode.cldr.util.CLDRInfo.UserInfo;
import org.unicode.cldr.util.CLDRLocale;
import org.unicode.cldr.util.CldrUtility;
import org.unicode.cldr.util.CoverageInfo;
import org.unicode.cldr.util.DateTimeFormats;
import org.unicode.cldr.util.DtdData.IllegalByDtdException;
import org.unicode.cldr.util.VoterReportStatus.ReportId;
import org.unicode.cldr.util.Factory;
import org.unicode.cldr.util.Level;
import org.unicode.cldr.util.PathHeader;
import org.unicode.cldr.util.SpecialLocales;
import org.unicode.cldr.util.SupplementalDataInfo;
import org.unicode.cldr.util.XMLSource;
import org.unicode.cldr.util.XMLUploader;
import org.unicode.cldr.util.XPathParts;
import org.unicode.cldr.web.BallotBox.InvalidXPathException;
import org.unicode.cldr.web.BallotBox.VoteNotAcceptedException;
import org.unicode.cldr.web.CLDRProgressIndicator.CLDRProgressTask;
Expand Down Expand Up @@ -1616,7 +1602,7 @@ private void importAnonymousOldLosingVote(BallotBox<User> box, CLDRLocale locale
/*
* Submit the anonymous vote.
*/
box.voteForValue(anonUser, xpathString, processedValue);
box.voteForValueWithType(anonUser, xpathString, processedValue, VoteType.MANUAL_IMPORT);
/*
* Add a row to the IMPORT table, to avoid importing the same value repeatedly.
* For this we need unprocessedValue, to match what occurs for the original votes in the
Expand Down Expand Up @@ -1907,7 +1893,7 @@ private int importAllOldWinningVotes(User user, SurveyMain sm, final String oldV
* "for a later version".
*/
if (box.getVoteValue(user, xpathString) == null) {
box.voteForValue(user, xpathString, value, BallotBox.VOTE_IS_AUTO_IMPORTED);
box.voteForValueWithType(user, xpathString, value, VoteType.AUTO_IMPORT);
confirmations++;
}
}
Expand Down
20 changes: 20 additions & 0 deletions tools/cldr-code/src/main/java/org/unicode/cldr/util/VoteType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package org.unicode.cldr.util;

public enum VoteType {
UNKNOWN(0), DIRECT(1), AUTO_IMPORT (2) , MANUAL_IMPORT(3), BULK_UPLOAD(4);

private final int integerId;

VoteType(int id) {
this.integerId = id;
}

/**
* Get an integer version of the type, for compact database storage as TINYINT
*
* @return the vote type integer id
*/
public int id() {
return integerId;
}
}