Skip to content

Commit

Permalink
Merge pull request #248 from ChrisJohnNOAA/rss_updates
Browse files Browse the repository at this point in the history 
Have file updates trigger RSS changes and Subscription actions.
  • Loading branch information
@ChrisJohnNOAA
ChrisJohnNOAA authored Jan 28, 2025
2 parents fe58ddb + 42805de commit d842433
Show file tree
Hide file tree
Showing 7 changed files with 338 additions and 23 deletions.
8 changes: 5 additions & 3 deletions WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,8 @@ public class Erddap extends HttpServlet {
new ConcurrentHashMap<>(16, 0.75f, 4);

/** The RSS info: key=datasetId, value=utf8 byte[] of rss xml */
public final ConcurrentHashMap<String, byte[]> rssHashMap = new ConcurrentHashMap<>(16, 0.75f, 4);
public static final ConcurrentHashMap<String, byte[]> rssHashMap =
new ConcurrentHashMap<>(16, 0.75f, 4);

public final ConcurrentHashMap<String, int[]> failedLogins =
new ConcurrentHashMap<>(16, 0.75f, 4);
Expand Down Expand Up @@ -23575,7 +23576,8 @@ public void processDataset(EDD dataset, SaxParsingContext context) {
* @param subject for email messages
* @param change the change description must be specified or nothing is done
*/
protected void tryToDoActions(String tDatasetID, EDD cooDataset, String subject, String change) {
public static void tryToDoActions(
String tDatasetID, EDD cooDataset, String subject, String change) {
if (String2.isSomething(tDatasetID) && String2.isSomething(change)) {
if (!String2.isSomething(subject)) subject = "Change to datasetID=" + tDatasetID;
try {
Expand Down Expand Up @@ -23654,7 +23656,7 @@ protected void tryToDoActions(String tDatasetID, EDD cooDataset, String subject,
// trigger RSS action
// (after new dataset is in place and if there is either a current or older dataset)
if (cooDataset != null) {
cooDataset.updateRSS(this, change);
cooDataset.updateRSS(change);
}

} catch (Throwable subT) {
Expand Down
93 changes: 91 additions & 2 deletions WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1245,6 +1245,95 @@ public String similar(EDD other) {
}
}

protected Map<String, String> snapshot() {
Map<String, String> snapshot = new HashMap<>();
snapshot.put("nDv", "" + dataVariables.length);

for (int dv = 0; dv < dataVariables.length; dv++) {
EDV variable = dataVariables()[dv];
snapshot.put("dv_" + dv + "_name", variable.destinationName());
snapshot.put("dv_" + dv + "_type", variable.destinationDataType());
snapshot.put("dv_" + dv + "_attr", variable.combinedAttributes().toString());
}

snapshot.put("globalAttributes", combinedGlobalAttributes.toString());

return snapshot;
}

/**
* This tests if 'oldSnapshot' is different from this in any way. <br>
* This test is from the view of a subscriber who wants to know when a dataset has changed in any
* way. <br>
* So some things like onChange and reloadEveryNMinutes are not checked. <br>
* This only lists the first change found.
*
* <p>EDDGrid overwrites this to also check the axis variables.
*
* @param old
* @return "" if same or message if not.
*/
public String changed(Map<String, String> oldSnapshot) {

// FUTURE: perhaps it would be nice if EDDTable changed showed new data.
// so it would appear in email subscription and rss.
// but for many datasets (e.g., ndbc met) there are huge number of buoys. so not practical.
if (oldSnapshot == null) return EDStatic.EDDChangedWasnt;

Map<String, String> newSnapshot = snapshot();
StringBuilder diff = new StringBuilder();
// check most important things first
if (!oldSnapshot.get("nDv").equals(newSnapshot.get("nDv"))) {
diff.append(
MessageFormat.format(
EDStatic.EDDChangedDifferentNVar, oldSnapshot.get("nDv"), newSnapshot.get("nDv")));
return diff.toString(); // because tests below assume nDv are same
}

int nDv = dataVariables.length;
for (int dv = 0; dv < nDv; dv++) {
String nameKey = "dv_" + dv + "_name";
String typeKey = "dv_" + dv + "_type";
String attrKey = "dv_" + dv + "_attr";
String msg2 = "#" + dv + "=" + newSnapshot.get(nameKey);
if (!oldSnapshot.get(nameKey).equals(newSnapshot.get(nameKey))) {
diff.append(
MessageFormat.format(
EDStatic.EDDChanged2Different,
"destinationName",
msg2,
oldSnapshot.get(nameKey),
newSnapshot.get(nameKey))
+ "\n");
}
if (!oldSnapshot.get(typeKey).equals(newSnapshot.get(typeKey))) {
diff.append(
MessageFormat.format(
EDStatic.EDDChanged2Different,
"destinationDataType",
msg2,
oldSnapshot.get(typeKey),
newSnapshot.get(typeKey))
+ "\n");
}
String s = String2.differentLine(oldSnapshot.get(attrKey), newSnapshot.get(attrKey));
if (s.length() > 0) {
diff.append(
MessageFormat.format(EDStatic.EDDChanged1Different, "combinedAttribute", msg2, s)
+ "\n");
}
}

// check least important things last
String s =
String2.differentLine(
oldSnapshot.get("globalAttributes"), newSnapshot.get("globalAttributes"));
if (s.length() > 0)
diff.append(MessageFormat.format(EDStatic.EDDChangedCGADifferent, s) + "\n");

return diff.toString();
}

// protected static String test1Changed(String msg, String diff) {
// return diff.length() == 0? "" : msg + "\n" + diff + "\n";
// }
Expand Down Expand Up @@ -1333,7 +1422,7 @@ public String changed(EDD old) {
* returns "")
* @return the rss document
*/
public String updateRSS(Erddap erddap, String change) {
public String updateRSS(String change) {
if (change == null || change.length() == 0) return "";
try {
// generate the rss xml
Expand Down Expand Up @@ -1376,7 +1465,7 @@ public String updateRSS(Erddap erddap, String change) {

// store the xml
String rssString = rss.toString();
if (erddap != null) erddap.rssHashMap.put(datasetID(), String2.stringToUtf8Bytes(rssString));
Erddap.rssHashMap.put(datasetID(), String2.stringToUtf8Bytes(rssString));
return rssString;

} catch (Throwable rssT) {
Expand Down
104 changes: 104 additions & 0 deletions WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGrid.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2613,6 +2613,110 @@ public String similar(EDDGrid other, int firstAxisToMatch, int matchAxisNDigits,
}
}

@Override
public Map<String, String> snapshot() {
Map<String, String> snapshot = super.snapshot();
snapshot.put("nAv", "" + dataVariables.length);
for (int av = 0; av < axisVariables.length; av++) {
EDVGridAxis variable = axisVariables()[av];
snapshot.put("av_" + av + "_name", variable.destinationName());
snapshot.put("av_" + av + "_type", variable.destinationDataType());
snapshot.put("av_" + av + "_sourceSize", "" + variable.sourceValues().size());
snapshot.put("av_" + av + "_minValue", "" + variable.destinationCoarseMin());
snapshot.put("av_" + av + "_maxValue", "" + variable.destinationCoarseMax());
snapshot.put("av_" + av + "_attr", variable.combinedAttributes().toString());
}

return snapshot;
}

@Override
public String changed(Map<String, String> oldSnapshot) {
if (oldSnapshot == null) return super.changed(oldSnapshot); // so message is consistent

Map<String, String> newSnapshot = snapshot();
StringBuilder diff = new StringBuilder();
// check most important things first
if (!oldSnapshot.get("nAv").equals(newSnapshot.get("nAv"))) {
diff.append(
MessageFormat.format(
EDStatic.EDDChangedAxesDifferentNVar,
oldSnapshot.get("nAv"),
newSnapshot.get("nAv")));
return diff.toString(); // because tests below assume nAv are same
}

int nAv = dataVariables.length;
for (int av = 0; av < nAv; av++) {
String nameKey = "av_" + av + "_name";
String typeKey = "av_" + av + "_type";
String sourceSizeKey = "av_" + av + "_sourceSize";
String minKey = "av_" + av + "_minValue";
String maxKey = "av_" + av + "_maxValue";
String attrKey = "av_" + av + "_attr";
String msg2 = "#" + av + "=" + newSnapshot.get(nameKey);
if (!oldSnapshot.get(nameKey).equals(newSnapshot.get(nameKey))) {
diff.append(
MessageFormat.format(
EDStatic.EDDChangedAxes2Different,
"destinationName",
msg2,
oldSnapshot.get(nameKey),
newSnapshot.get(nameKey))
+ "\n");
}
if (!oldSnapshot.get(typeKey).equals(newSnapshot.get(typeKey))) {
diff.append(
MessageFormat.format(
EDStatic.EDDChangedAxes2Different,
"destinationDataType",
msg2,
oldSnapshot.get(typeKey),
newSnapshot.get(typeKey))
+ "\n");
}
if (!oldSnapshot.get(sourceSizeKey).equals(newSnapshot.get(sourceSizeKey))) {
diff.append(
MessageFormat.format(
EDStatic.EDDChangedAxes2Different,
"numberOfValues",
msg2,
oldSnapshot.get(sourceSizeKey),
newSnapshot.get(sourceSizeKey))
+ "\n");
}
if (!oldSnapshot.get(minKey).equals(newSnapshot.get(minKey))) {
diff.append(
MessageFormat.format(
EDStatic.EDDChangedAxes2Different,
"minValue",
msg2,
oldSnapshot.get(minKey),
newSnapshot.get(minKey))
+ "\n");
}
if (!oldSnapshot.get(maxKey).equals(newSnapshot.get(maxKey))) {
diff.append(
MessageFormat.format(
EDStatic.EDDChangedAxes2Different,
"maxValue",
msg2,
oldSnapshot.get(maxKey),
newSnapshot.get(maxKey))
+ "\n");
}
String s = String2.differentLine(oldSnapshot.get(attrKey), newSnapshot.get(attrKey));
if (s.length() > 0) {
diff.append(
MessageFormat.format(EDStatic.EDDChangedAxes1Different, "combinedAttribute", msg2, s)
+ "\n");
}
}

diff.append(super.changed(oldSnapshot));
return diff.toString();
}

/**
* This tests if 'old' is different from this in any way. <br>
* This test is from the view of a subscriber who wants to know when a dataset has changed in any
Expand Down
13 changes: 5 additions & 8 deletions WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromFiles.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import java.util.ArrayList;
import java.util.BitSet;
import java.util.HashSet;
import java.util.Map;
import java.util.TimeZone;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
Expand Down Expand Up @@ -1816,6 +1817,7 @@ private boolean handleEventContexts(StringArray contexts, String msg) throws Thr
requestReloadASAP();
return false;
}
Map<String, String> snapshot = snapshot();

// get BadFile and FileTable info and make local copies
ConcurrentHashMap badFileMap = readBadFileMap(); // already a copy of what's in file
Expand Down Expand Up @@ -2057,14 +2059,9 @@ private boolean handleEventContexts(StringArray contexts, String msg) throws Thr
}

// after changes all in place
// Currently, update() doesn't trigger these changes.
// The problem is that some datasets might update every second, others every
// day.
// Even if they are done, perhaps do them in ERDDAP ((low)update return
// changes?)
// ?update rss?
// ?subscription and onchange actions?

if (EDStatic.updateSubsRssOnFileChanges) {
Erddap.tryToDoActions(datasetID(), this, "", changed(snapshot));
}
}

if (verbose)
Expand Down
12 changes: 4 additions & 8 deletions WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromFiles.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3210,6 +3210,7 @@ private boolean handleEventContexts(StringArray contexts, String msg) throws Thr
requestReloadASAP();
return false;
}
Map<String, String> snapshot = snapshot();

// get BadFile and FileTable info and make local copies
ConcurrentHashMap badFileMap = readBadFileMap(); // already a copy of what's in file
Expand Down Expand Up @@ -3435,14 +3436,9 @@ private boolean handleEventContexts(StringArray contexts, String msg) throws Thr
}

// after changes all in place
// Currently, update() doesn't trigger these changes.
// The problem is that some datasets might update every second, others every
// day.
// Even if they are done, perhaps do them in ERDDAP ((low)update return
// changes?)
// ?update rss?
// ?subscription and onchange actions?

if (EDStatic.updateSubsRssOnFileChanges) {
Erddap.tryToDoActions(datasetID(), this, "", changed(snapshot));
}
}

if (verbose)
Expand Down
2 changes: 2 additions & 0 deletions WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -810,6 +810,7 @@ public static int convertToPublicSourceUrlFromSlashPo(String tFrom) {
public static final boolean variablesMustHaveIoosCategory;
public static boolean verbose;
public static boolean useSaxParser;
public static boolean updateSubsRssOnFileChanges;
public static final boolean useEddReflection;
public static boolean enableCors;
public static final String corsAllowHeaders;
Expand Down Expand Up @@ -2304,6 +2305,7 @@ public static int convertToPublicSourceUrlFromSlashPo(String tFrom) {
subscriptionSystemActive = getSetupEVBoolean(setup, ev, "subscriptionSystemActive", true);
convertersActive = getSetupEVBoolean(setup, ev, "convertersActive", true);
useSaxParser = getSetupEVBoolean(setup, ev, "useSaxParser", false);
updateSubsRssOnFileChanges = getSetupEVBoolean(setup, ev, "updateSubsRssOnFileChanges", true);
useEddReflection = getSetupEVBoolean(setup, ev, "useEddReflection", false);
enableCors = getSetupEVBoolean(setup, ev, "enableCors", false);
corsAllowHeaders =
Expand Down
Loading

0 comments on commit d842433

Please sign in to comment.