From 54a350cf4cc0f824f9b9891ecf14912613559336 Mon Sep 17 00:00:00 2001 From: Brett Johnson Date: Thu, 4 May 2017 16:14:32 -0700 Subject: [PATCH] Remove DateFilter from 4.1.3 (#40) --- .../adaptor/prebuilt/DateFilter.java | 397 -------------- .../adaptor/prebuilt/DateFilterTest.java | 492 ------------------ 2 files changed, 889 deletions(-) delete mode 100644 src/com/google/enterprise/adaptor/prebuilt/DateFilter.java delete mode 100644 test/com/google/enterprise/adaptor/prebuilt/DateFilterTest.java diff --git a/src/com/google/enterprise/adaptor/prebuilt/DateFilter.java b/src/com/google/enterprise/adaptor/prebuilt/DateFilter.java deleted file mode 100644 index 3985db22..00000000 --- a/src/com/google/enterprise/adaptor/prebuilt/DateFilter.java +++ /dev/null @@ -1,397 +0,0 @@ -// Copyright 2017 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.enterprise.adaptor.prebuilt; - -import static com.google.enterprise.adaptor.MetadataTransform.TransmissionDecision; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.enterprise.adaptor.Metadata; -import com.google.enterprise.adaptor.MetadataTransform; - -import java.text.DateFormat; -import java.text.FieldPosition; -import java.text.ParseException; -import java.text.ParsePosition; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Transform causing exclusion of certain Documents, based on a date - * in that document's Metadata or Param properties. Documents whose - * associated date is before the configured date will not be indexed. - *

- * Configuration properties: - *

- * {@code key} The name of the metadata or param property whose date value - * determines whether the document will be skipped or not. - *

- * {@code format} The {@link DateFormat} used to parse the document's - * date values. If no {@code format} is specified, a ISO8601 format - * ("yyyy-MM-dd") is used. If format is "millis", then the dates - * are parsed as if they were milliseconds since the epoch - * (January 1, 1970, 00:00:00 GMT). Otherwise, a lenient - * {@link SimpleDateFormat} with the format pattern will be used. - *

- * {@code date} The cut-off for the date value. Document's whose date - * value is before the configued {@code date} will not be indexed. - * The configured {@code date} must be parsable by the configured date - * {@code format}, unless {@code format} is "millis", in which - * case the date must be specified in ISO8601 format. Only one of - * {@code date} or {@code days} configuration may be specified. - *

- * {@code days} The cut-off for the date value. Document's whose date - * value is more than {@code days} before present will not be indexed. - * This can be used to have a rolling window of indexed documents. Those - * whose date have expired will be removed from the index. Only one of - * {@code date} or {@code days} configuration may be specified. - *

- * {@code corpora} The location of the date value to consider. The - * {@code corpora} may be set to {@code metadata} or to {@code params} - * to restrict the search to only metadata or params, respectively, - * or to {@code metadata or params} to search both. - *

- * Example 1: skip documents that have not been accessed for more than 3 years: - * 


-   metadata.transform.pipeline=dateFilter
-   metadata.transform.pipeline.dateFilter.factoryMethod=com.google.enterprise.adaptor.prebuilt.DateFilter.create
-   metadata.transform.pipeline.dateFilter.key=Last_Access_Date
-   metadata.transform.pipeline.dateFilter.days=1095
-
- *

- * Example 2: skip pre-Y2K client records that used old-style US date formats: - * 


-   metadata.transform.pipeline=dateFilter
-   metadata.transform.pipeline.dateFilter.factoryMethod=com.google.enterprise.adaptor.prebuilt.DateFilter.create
-   metadata.transform.pipeline.dateFilter.key=Last_Visit_Date
-   metadata.transform.pipeline.dateFilter.format=MM/dd/YY
-   metadata.transform.pipeline.dateFilter.date=01/01/00
-
- *

- * Example 3: skip documents that have not been modified since 2010: - * 


-   metadata.transform.pipeline=dateFilter
-   metadata.transform.pipeline.dateFilter.factoryMethod=com.google.enterprise.adaptor.prebuilt.DateFilter.create
-   metadata.transform.pipeline.dateFilter.corpora=params
-   metadata.transform.pipeline.dateFilter.key=Last-Modified-Millis-UTC
-   metadata.transform.pipeline.dateFilter.format=millis
-   metadata.transform.pipeline.dateFilter.date=2010-01-01
-
- */ -public class DateFilter implements MetadataTransform { - private static final Logger log - = Logger.getLogger(DateFilter.class.getName()); - - private static final String ISO_8601_FORMAT = "yyyy-MM-dd"; - - /** - * Which collections of keys/values to search. Metadata, params, or both. - */ - private enum Corpora { - METADATA("metadata"), - PARAMS("params"), - METADATA_OR_PARAMS("metadata or params"); - - private final String name; - - private Corpora(String n) { - name = n; - } - - public static Corpora from(String val) { - if ("metadata".equalsIgnoreCase(val)) { - return Corpora.METADATA; - } - if ("params".equalsIgnoreCase(val)) { - return Corpora.PARAMS; - } - return METADATA_OR_PARAMS; - } - - public String toString() { - return name; - } - }; - - /** The name of the key (either Metadata key or params key) to match. */ - private final String key; - - /** The DateFormat used to parse the date values. */ - private final String dateFormatString; - private final ThreadLocal dateFormat; - - /** The active DateValueFilter */ - private final DateValueFilter filter; - - /** - * If {@code METADATA}, only search the metadata for the specified key; - * if {@code PARAMS}, only search the params for the specified key; - * if {@code METADATA_OR_PARAMS}, search both. - */ - private Corpora corpora = Corpora.METADATA_OR_PARAMS; - - private DateFilter(String key, String dateFormatStr, DateValueFilter filter, - Corpora corpora) { - this.key = key; - this.dateFormatString = dateFormatStr; - this.filter = filter; - this.corpora = corpora; - - // DateFormat is not thread-safe, so each thread gets its own instance. - this.dateFormat = new ThreadLocal() { - @Override - protected DateFormat initialValue() { - DateFormat format; - if ("millis".equalsIgnoreCase(dateFormatString)) { - format = new MillisecondDateFormat(); - } else { - format = new SimpleDateFormat(dateFormatString); - format.setLenient(true); - } - return format; - } - }; - } - - /** - * Search (only) the {@code Metadata} for an instance of the key - * containing a date value that would trigger this document to be - * skipped. - * Returns a date value that the filter would exclude, or null - * if none match. - */ - private String excludedByDateInMetadata(Metadata metadata) { - for (String value : metadata.getAllValues(key)) { - try { - if (filter.excluded(dateFormat.get().parse(value))) { - return value; - } - } catch (ParseException e) { - log.log(Level.WARNING, "Date value " + value + " does not conform to " - + "date format " + dateFormatString, e); - } - } - return null; - } - - /** - * Search (only) the {@code params} for an instance of the key - * containing a date value that would trigger this document to be - * skipped. - * Returns a date value that the filter would exclude, or null - * if none match. - */ - private String excludedByDateInParams(Map params) { - String value = params.get(key); - if (value != null) { - try { - if (filter.excluded(dateFormat.get().parse(value))) { - return value; - } - } catch (ParseException e) { - log.log(Level.WARNING, "Date value " + value + " does not conform to " - + "date format " + dateFormatString, e); - } - } - return null; - } - - /** - * Conditionally adds a {@code Transmission-Decision} entry to the - * {@code params Map}. The decision is based on settings of the - * {@code key}, {@code pattern}, {@code decideOnMatch}, {@code decision}, - * and {@code corpora} configuration variables (as discussed above). - */ - @Override - public void transform(Metadata metadata, Map params) { - String excludedDate; - - switch (corpora) { - case METADATA: - excludedDate = excludedByDateInMetadata(metadata); - break; - case PARAMS: - excludedDate = excludedByDateInParams(params); - break; - case METADATA_OR_PARAMS: - default: - excludedDate = excludedByDateInParams(params); - if (excludedDate == null) { - excludedDate = excludedByDateInMetadata(metadata); - } - } - - String docId = params.get(MetadataTransform.KEY_DOC_ID); - if (Strings.isNullOrEmpty(docId)) { - docId = "with no docId"; - } - - if (excludedDate != null) { - log.log(Level.INFO, "Skipping document {0}, because {1}: {2} is too old.", - new Object[] { docId, key, excludedDate }); - params.put(MetadataTransform.KEY_TRANSMISSION_DECISION, - TransmissionDecision.DO_NOT_INDEX.toString()); - } else { - log.log(Level.FINE, "Not skipping document {0}, because {1} is in range.", - new Object[] { docId, key }); - } - } - - @Override - public String toString() { - return new StringBuilder("DateFilter(") - .append(key).append(", ") - .append(dateFormatString).append(", ") - .append(filter.toString()).append(", ") - .append(corpora).append(")") - .toString(); - } - - public static DateFilter create(Map cfg) { - String key; - String format; - DateValueFilter filter; - Corpora corpora; - - key = getTrimmedValue(cfg, "key"); - if (key == null) { - throw new NullPointerException("key may not be null or empty"); - } - log.config("key = " + key); - - format = getTrimmedValue(cfg, "format"); - if (format == null) { - format = ISO_8601_FORMAT; - } - log.config("format = " + format); - // If using MillisecondDateFormat, the cutoff date is specified as ISO8601, - // otherwise the cutoff date is in the configured format. - SimpleDateFormat dateFormat = new SimpleDateFormat( - "millis".equalsIgnoreCase(format) ? ISO_8601_FORMAT : format); - dateFormat.setLenient(true); - - String dateStr = getTrimmedValue(cfg, "date"); - String daysStr = getTrimmedValue(cfg, "days"); - if (dateStr != null) { - if (daysStr != null) { - throw new IllegalArgumentException("Only one of 'date' or 'days' " - + " configuration may be specified."); - } - try { - filter = - new AbsoluteDateValueFilter(dateFormat.parse(dateStr), dateStr); - } catch (ParseException e) { - throw new IllegalArgumentException("date " + dateStr - + " does not conform to date format " + format, e); - } - log.config("date = " + dateStr); - } else if (daysStr != null) { - filter = new ExpiringDateValueFilter(Integer.parseInt(daysStr)); - log.config("days = " + daysStr); - } else { - throw new IllegalArgumentException("Either 'date' or 'days' " - + " configuration must be specified."); - } - log.log(Level.INFO, "Documents whose {0} date is earlier than {1} will be " - + "skipped.", new Object[] { key, filter.toString() }); - - corpora = Corpora.from(getTrimmedValue(cfg, "corpora")); - log.config("corpora set to " + corpora); - - return new DateFilter(key, format, filter, corpora); - } - - private static String getTrimmedValue(Map cfg, String key) { - String value = cfg.get(key); - return (value == null) ? value : Strings.emptyToNull(value.trim()); - } - - /** A DateFormat that parses text of milliseconds since the epoch. */ - @VisibleForTesting - static class MillisecondDateFormat extends DateFormat { - @Override - public StringBuffer format(Date date, StringBuffer buf, FieldPosition pos) { - buf.append(Long.toString(date.getTime())); - return buf; - } - - @Override - public Date parse(String source, ParsePosition pos) { - try { - long millis = Long.parseLong(source); - pos.setIndex(source.length()); - return new Date(millis); - } catch (NumberFormatException e) { - pos.setErrorIndex(pos.getIndex()); - return null; - } - } - } - - private static interface DateValueFilter { - public boolean excluded(Date date); - } - - private static class AbsoluteDateValueFilter implements DateValueFilter { - private final Date oldestAllowed; - private final String dateStr; - - public AbsoluteDateValueFilter(Date oldestAllowed, String dateStr) { - Preconditions.checkArgument(oldestAllowed.compareTo(new Date()) < 0, - oldestAllowed.toString() + " is in the future."); - this.oldestAllowed = oldestAllowed; - this.dateStr = dateStr; - } - - @Override - public boolean excluded(Date date) { - return date.compareTo(oldestAllowed) < 0; - } - - @Override - public String toString() { - return dateStr; - } - } - - private static class ExpiringDateValueFilter implements DateValueFilter { - private final int daysOld; - private final long relativeMillis; - - public ExpiringDateValueFilter(int daysOld) { - Preconditions.checkArgument(daysOld > 0, "The number of days old for " - + "expired content must be greater than zero."); - this.daysOld = daysOld; - this.relativeMillis = TimeUnit.DAYS.toMillis(daysOld); - } - - @Override - public boolean excluded(Date date) { - Date oldestAllowed - = new Date(System.currentTimeMillis() - relativeMillis); - return date.compareTo(oldestAllowed) < 0; - } - - @Override - public String toString() { - return daysOld + " days"; - } - } -} diff --git a/test/com/google/enterprise/adaptor/prebuilt/DateFilterTest.java b/test/com/google/enterprise/adaptor/prebuilt/DateFilterTest.java deleted file mode 100644 index 6b5fa916..00000000 --- a/test/com/google/enterprise/adaptor/prebuilt/DateFilterTest.java +++ /dev/null @@ -1,492 +0,0 @@ -// Copyright 2017 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.enterprise.adaptor.prebuilt; - -import static org.junit.Assert.assertEquals; - -import com.google.enterprise.adaptor.Metadata; -import com.google.enterprise.adaptor.MetadataTransform; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -/** Unit tests for {@link DateFilter}. */ -public class DateFilterTest { - @Rule - public ExpectedException thrown = ExpectedException.none(); - - private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - - @Test - public void testMillisecondDateFormat() throws Exception { - DateFormat msDateFormat = new DateFilter.MillisecondDateFormat(); - Date now = new Date(); - Date msDate = msDateFormat.parse(Long.toString(now.getTime())); - assertEquals(now, msDate); - assertEquals(Long.toString(now.getTime()), msDateFormat.format(now)); - } - - // tests on create calls (various errors) and toString results - @Test - public void testCreate_NoKey() { - thrown.expect(NullPointerException.class); - Map config = new HashMap(); - config.put("days", "365"); - DateFilter transform = DateFilter.create(config); - } - - @Test - public void testCreate_EmptyKey() { - thrown.expect(NullPointerException.class); - Map config = new HashMap(); - config.put("key", ""); - config.put("days", "365"); - DateFilter transform = DateFilter.create(config); - } - - @Test - public void testCreate_InvalidFormat() { - thrown.expect(IllegalArgumentException.class); - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("format", "bogus"); - config.put("days", "365"); - DateFilter transform = DateFilter.create(config); - } - - @Test - public void testCreate_DefaultFormat() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "365"); - DateFilter transform = DateFilter.create(config); - assertEquals("DateFilter(lastModified, yyyy-MM-dd, 365 days, " - + "metadata or params)", transform.toString()); - } - - @Test - public void testCreate_EmptyFormatSameAsNull() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("format", ""); - config.put("days", "365"); - DateFilter transform = DateFilter.create(config); - assertEquals("DateFilter(lastModified, yyyy-MM-dd, 365 days, " - + "metadata or params)", transform.toString()); - } - - @Test - public void testMillisecondFormat() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("format", "millis"); - config.put("date", "2000-01-01"); - DateFilter transform = DateFilter.create(config); - assertEquals("DateFilter(lastModified, millis, 2000-01-01, " - + "metadata or params)", transform.toString()); - } - - @Test - public void testCreate_NoDateOrDays() { - thrown.expect(IllegalArgumentException.class); - Map config = new HashMap(); - config.put("key", "lastModified"); - DateFilter transform = DateFilter.create(config); - } - - @Test - public void testCreate_BothDateAndDays() { - thrown.expect(IllegalArgumentException.class); - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("date", "1977-06-18"); - config.put("days", "365"); - DateFilter transform = DateFilter.create(config); - } - - @Test - public void testCreate_DateDoesNotMatchFormat() { - thrown.expect(IllegalArgumentException.class); - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("format", "yyyy-MM-dd"); - config.put("date", "6/18/1977"); - DateFilter transform = DateFilter.create(config); - } - - @Test - public void testCreate_DateInTheFuture() { - thrown.expect(IllegalArgumentException.class); - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("date", "2525-06-18"); - DateFilter transform = DateFilter.create(config); - } - - @Test - public void testCreate_DaysInTheFuture() { - thrown.expect(IllegalArgumentException.class); - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "-18"); - DateFilter transform = DateFilter.create(config); - } - - @Test - public void testToString_CustomFormat() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("format", "yyMMddHHmmssZ"); - config.put("days", "365"); - MetadataTransform transform = DateFilter.create(config); - assertEquals("DateFilter(lastModified, yyMMddHHmmssZ, 365 days, " - + "metadata or params)", transform.toString()); - } - - @Test - public void testToString_DateValueFilter() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("date", "1977-06-18"); - MetadataTransform transform = DateFilter.create(config); - assertEquals("DateFilter(lastModified, yyyy-MM-dd, 1977-06-18, " - + "metadata or params)", transform.toString()); - } - - @Test - public void testToString_CorporaMetadata() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "365"); - config.put("corpora", "metadata"); - MetadataTransform transform = DateFilter.create(config); - assertEquals("DateFilter(lastModified, yyyy-MM-dd, 365 days, " - + "metadata)", transform.toString()); - } - - @Test - public void testToString_CorporaParams() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "365"); - config.put("corpora", "params"); - MetadataTransform transform = DateFilter.create(config); - assertEquals("DateFilter(lastModified, yyyy-MM-dd, 365 days, " - + "params)", transform.toString()); - } - - @Test - public void testCreate_CorporaBogus() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "365"); - config.put("corpora", "bogus"); - MetadataTransform transform = DateFilter.create(config); - assertEquals("DateFilter(lastModified, yyyy-MM-dd, 365 days, " - + "metadata or params)", transform.toString()); - } - - - // tests on transform behavior - - @Test - public void testTransform_DoNotSkipUndatedDocument() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "365"); - config.put("corpora", "params"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - Metadata metadata = new Metadata(); - transform.transform(metadata, params); - assertEquals(null, params.get("Transmission-Decision")); - } - - @Test - public void testTransform_RelativeDoNotSkipNewDocument() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "365"); - config.put("corpora", "params"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - params.put("lastModified", dateFormat.format(new Date())); - Metadata metadata = new Metadata(); - transform.transform(metadata, params); - assertEquals(null, params.get("Transmission-Decision")); - } - - @Test - public void testTransform_RelativeSkipOldDocument() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "365"); - config.put("corpora", "params"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - params.put("lastModified", "1977-06-18"); - params.put(MetadataTransform.KEY_DOC_ID, "docId01"); - Metadata metadata = new Metadata(); - transform.transform(metadata, params); - assertEquals("do-not-index", params.get("Transmission-Decision")); - } - - @Test - public void testTransform_RelativeWindow() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "5"); - config.put("corpora", "params"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - params.put("lastModified", dateFormat.format(new Date( - System.currentTimeMillis() - TimeUnit.DAYS.toMillis(4)))); - Metadata metadata = new Metadata(); - transform.transform(metadata, params); - assertEquals(null, params.get("Transmission-Decision")); - - params.put("lastModified", dateFormat.format(new Date( - System.currentTimeMillis() - TimeUnit.DAYS.toMillis(6)))); - transform.transform(metadata, params); - assertEquals("do-not-index", params.get("Transmission-Decision")); - } - - @Test - public void testTransform_AbsoluteDoNotSkipNewDocument() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("date", "1977-06-18"); - config.put("corpora", "params"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - params.put("lastModified", dateFormat.format(new Date())); - Metadata metadata = new Metadata(); - transform.transform(metadata, params); - assertEquals(null, params.get("Transmission-Decision")); - } - - @Test - public void testTransform_AbsoluteSkipOldDocument() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("date", "1977-06-18"); - config.put("corpora", "params"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - params.put("lastModified", "1969-07-20"); - Metadata metadata = new Metadata(); - transform.transform(metadata, params); - assertEquals("do-not-index", params.get("Transmission-Decision")); - } - - @Test - public void testTransform_AbsoluteWindow() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("date", "1977-06-18"); - config.put("corpora", "params"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - params.put("lastModified", "1977-06-18"); - Metadata metadata = new Metadata(); - transform.transform(metadata, params); - assertEquals(null, params.get("Transmission-Decision")); - - params.put("lastModified", "1977-06-17"); - transform.transform(metadata, params); - assertEquals("do-not-index", params.get("Transmission-Decision")); - } - - @Test - public void testTransform_LenientDateParsing() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("format", "yyyy-MM-dd"); - config.put("days", "365"); - config.put("corpora", "params"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - params.put("lastModified", "1969-07-21T02:56:15Z"); - Metadata metadata = new Metadata(); - transform.transform(metadata, params); - assertEquals("do-not-index", params.get("Transmission-Decision")); - } - - @Test - public void testTransform_CustomDateParsing() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("format", "MM/dd/yyyy"); - config.put("days", "365"); - config.put("corpora", "params"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - params.put("lastModified", "07/21/1969"); - Metadata metadata = new Metadata(); - transform.transform(metadata, params); - assertEquals("do-not-index", params.get("Transmission-Decision")); - } - - @Test - public void testTransform_MillisecondDateParsing() throws Exception { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("format", "millis"); - config.put("days", "365"); - config.put("corpora", "params"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - params.put("lastModified", Long.toString(new Date().getTime())); - Metadata metadata = new Metadata(); - transform.transform(metadata, params); - assertEquals(null, params.get("Transmission-Decision")); - params.put("lastModified", - Long.toString(dateFormat.parse("1977-06-17").getTime())); - transform.transform(metadata, params); - assertEquals("do-not-index", params.get("Transmission-Decision")); - } - - @Test - public void testTransform_DoNotSkipUnparsableDate() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "365"); - config.put("corpora", "params"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - params.put("lastModified", "07/21/1969"); - Metadata metadata = new Metadata(); - transform.transform(metadata, params); - assertEquals(null, params.get("Transmission-Decision")); - } - - @Test - public void testTransform_DateInMetadata() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "365"); - config.put("corpora", "metadata"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - Metadata metadata = new Metadata(); - metadata.add("lastModified", "1977-06-18"); - transform.transform(metadata, params); - assertEquals("do-not-index", params.get("Transmission-Decision")); - } - - @Test - public void testTransform_RecentDateInMetadata() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "365"); - config.put("corpora", "metadata"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - Metadata metadata = new Metadata(); - metadata.add("lastModified", dateFormat.format(new Date())); - transform.transform(metadata, params); - assertEquals(null, params.get("Transmission-Decision")); - } - - @Test - public void testTransform_MultipleDatesInMetadata() { - Map config = new HashMap(); - config.put("key", "modifiedHistory"); - config.put("days", "365"); - config.put("corpora", "metadata"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - Metadata metadata = new Metadata(); - metadata.add("modifiedHistory", dateFormat.format(new Date())); - metadata.add("modifiedHistory", "1977-06-18"); - transform.transform(metadata, params); - assertEquals("do-not-index", params.get("Transmission-Decision")); - } - - @Test - public void testTransform_MultipleDatesInMetadataOneBad() { - Map config = new HashMap(); - config.put("key", "modifiedHistory"); - config.put("days", "365"); - config.put("corpora", "metadata"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - Metadata metadata = new Metadata(); - metadata.add("modifiedHistory", "07/21/1969"); - metadata.add("modifiedHistory", "1977-06-18"); - transform.transform(metadata, params); - assertEquals("do-not-index", params.get("Transmission-Decision")); - } - - @Test - public void testTransform_DateInMetadataOrParams1() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "365"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - Metadata metadata = new Metadata(); - metadata.add("lastModified", "1977-06-18"); - transform.transform(metadata, params); - assertEquals("do-not-index", params.get("Transmission-Decision")); - } - - @Test - public void testTransform_DateInMetadataOrParams2() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "365"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - params.put("lastModified", "1977-06-18"); - Metadata metadata = new Metadata(); - transform.transform(metadata, params); - assertEquals("do-not-index", params.get("Transmission-Decision")); - } - - @Test - public void testTransform_DateInMetadataAndParams() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "365"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - params.put("lastModified", dateFormat.format(new Date())); - Metadata metadata = new Metadata(); - metadata.add("lastModified", "1977-06-18"); - transform.transform(metadata, params); - assertEquals("do-not-index", params.get("Transmission-Decision")); - } - - @Test - public void testTransform_RecentDateInMetadataAndParams() { - Map config = new HashMap(); - config.put("key", "lastModified"); - config.put("days", "365"); - DateFilter transform = DateFilter.create(config); - Map params = new HashMap(); - params.put("lastModified", dateFormat.format(new Date())); - Metadata metadata = new Metadata(); - metadata.add("lastModified", dateFormat.format(new Date())); - transform.transform(metadata, params); - assertEquals(null, params.get("Transmission-Decision")); - } -}