From 9771f23dcd204d0fed207505c485be2a73d70694 Mon Sep 17 00:00:00 2001
From: Alex Osborne <aosborne@nla.gov.au>
Date: Fri, 28 Jun 2024 15:32:16 +0900
Subject: [PATCH] Add lenient WARC parsing mode

Fixes #26 and #86
---
 src/org/netpreserve/jwarc/WarcParser.java     | 380 ++++++++++++++++--
 src/org/netpreserve/jwarc/WarcParser.rl       |  41 ++
 src/org/netpreserve/jwarc/WarcReader.java     |  15 +
 src/org/netpreserve/jwarc/cdx/CdxWriter.java  |   1 +
 .../org/netpreserve/jwarc/WarcParserTest.java |  16 +
 5 files changed, 413 insertions(+), 40 deletions(-)

diff --git a/src/org/netpreserve/jwarc/WarcParser.java b/src/org/netpreserve/jwarc/WarcParser.java
index 0ae73d4..dc3a5d4 100644
--- a/src/org/netpreserve/jwarc/WarcParser.java
+++ b/src/org/netpreserve/jwarc/WarcParser.java
@@ -21,7 +21,7 @@
 import static java.nio.charset.StandardCharsets.US_ASCII;
 
 
-// line 169 "WarcParser.rl"
+// line 185 "WarcParser.rl"
 
 
 /**
@@ -71,6 +71,30 @@ public void reset() {
         }
     }
 
+	/**
+	 * Sets the lenient mode for the WarcParser.
+	 * <p>
+	 * When enabled, this causes the parser to follow the specification less strictly,
+	 * allowing reading of non-compliant records by:
+	 * <ul>
+	 *   <li>permitting ASCII control characters in header field names and values
+	 *   <li>allowing lines to end with LF instead of CRLF
+	 * <ul>
+	 * Calling this method also resets the state of the parser.
+	 */
+    public void setLenient(boolean lenient) {
+        if (warcFieldsMode()) {
+            entryState = lenient ? warc_en_warc_fields_lenient : warc_en_warc_fields;
+        } else {
+            entryState = lenient ? warc_en_any_header_lenient : warc_start;
+        }
+        reset();
+    }
+
+    private boolean warcFieldsMode() {
+        return entryState == warc_en_warc_fields || entryState == warc_en_warc_fields_lenient;
+    }
+
     public boolean isFinished() {
         return cs >= warc_first_final;
     }
@@ -84,7 +108,7 @@ public void parse(ByteBuffer data) {
         int pe = data.limit();
 
         
-// line 88 "WarcParser.java"
+// line 112 "WarcParser.java"
 	{
 	int _klen;
 	int _trans = 0;
@@ -275,10 +299,14 @@ else if ( ( (data.get(p) & 0xff)) > _warc_trans_keys[_mid+1] )
 }
 	break;
 	case 13:
-// line 167 "WarcParser.rl"
+// line 182 "WarcParser.rl"
+	{ { p += 1; _goto_targ = 5; if (true)  continue _goto;} }
+	break;
+	case 14:
+// line 183 "WarcParser.rl"
 	{ { p += 1; _goto_targ = 5; if (true)  continue _goto;} }
 	break;
-// line 282 "WarcParser.java"
+// line 310 "WarcParser.java"
 			}
 		}
 	}
@@ -298,7 +326,7 @@ else if ( ( (data.get(p) & 0xff)) > _warc_trans_keys[_mid+1] )
 	break; }
 	}
 
-// line 231 "WarcParser.rl"
+// line 271 "WarcParser.rl"
 
         position += p - data.position();
         data.position(p);
@@ -352,14 +380,14 @@ private void setHeader(String name, String value) {
     }
 
     
-// line 356 "WarcParser.java"
+// line 384 "WarcParser.java"
 private static byte[] init__warc_actions_0()
 {
 	return new byte [] {
 	    0,    1,    0,    1,    1,    1,    2,    1,    3,    1,    4,    1,
 	    5,    1,    6,    1,    7,    1,    8,    1,    9,    1,   11,    1,
-	   13,    2,    3,    0,    2,    4,    0,    2,    6,    0,    3,   10,
-	   12,   13
+	   13,    1,   14,    2,    3,    0,    2,    4,    0,    2,    6,    0,
+	    2,    6,   13,    3,   10,   12,   13,    3,   10,   12,   14
 	};
 }
 
@@ -383,8 +411,23 @@ private static short[] init__warc_key_offsets_0()
 	  993,  996,  999, 1002, 1005, 1006, 1008, 1011, 1013, 1016, 1018, 1021,
 	 1024, 1025, 1028, 1029, 1032, 1033, 1036, 1037, 1040, 1041, 1042, 1043,
 	 1044, 1046, 1049, 1051, 1054, 1055, 1071, 1072, 1088, 1094, 1095, 1113,
-	 1119, 1125, 1131, 1147, 1148, 1164, 1170, 1171, 1189, 1195, 1201, 1207,
-	 1207
+	 1119, 1125, 1131, 1136, 1139, 1143, 1147, 1151, 1156, 1160, 1164, 1168,
+	 1170, 1186, 1187, 1203, 1209, 1210, 1228, 1234, 1240, 1246, 1250, 1253,
+	 1256, 1258, 1261, 1263, 1266, 1270, 1272, 1274, 1276, 1278, 1280, 1282,
+	 1284, 1286, 1289, 1306, 1308, 1310, 1313, 1316, 1320, 1322, 1324, 1326,
+	 1328, 1330, 1333, 1335, 1337, 1339, 1342, 1358, 1375, 1394, 1398, 1403,
+	 1406, 1423, 1439, 1454, 1472, 1479, 1482, 1486, 1504, 1521, 1538, 1555,
+	 1573, 1590, 1608, 1625, 1634, 1645, 1657, 1669, 1682, 1691, 1700, 1709,
+	 1718, 1729, 1741, 1750, 1759, 1770, 1782, 1786, 1790, 1795, 1813, 1830,
+	 1848, 1865, 1884, 1890, 1894, 1898, 1903, 1921, 1938, 1955, 1973, 1990,
+	 2009, 2014, 2018, 2022, 2026, 2028, 2030, 2034, 2038, 2042, 2048, 2052,
+	 2056, 2060, 2078, 2096, 2114, 2131, 2150, 2156, 2160, 2164, 2168, 2172,
+	 2176, 2178, 2180, 2182, 2188, 2194, 2197, 2200, 2203, 2206, 2209, 2212,
+	 2215, 2218, 2221, 2224, 2227, 2230, 2233, 2236, 2239, 2242, 2245, 2248,
+	 2251, 2252, 2254, 2257, 2259, 2262, 2264, 2267, 2270, 2271, 2274, 2275,
+	 2278, 2279, 2282, 2283, 2286, 2287, 2288, 2289, 2290, 2292, 2295, 2297,
+	 2301, 2306, 2309, 2313, 2317, 2321, 2326, 2330, 2334, 2338, 2340, 2342,
+	 2342, 2342, 2342
 	};
 }
 
@@ -488,13 +531,108 @@ private static char[] init__warc_trans_keys_0()
 	    0,   31,   10,    9,   13,   32,   33,  124,  126,   35,   39,   42,
 	   43,   45,   46,   48,   57,   65,   90,   94,  122,    9,   13,   32,
 	  127,    0,   31,    9,   13,   32,  127,    0,   31,    9,   13,   32,
-	  127,    0,   31,   13,   33,  124,  126,   35,   39,   42,   43,   45,
-	   46,   48,   57,   65,   90,   94,  122,   10,   33,   58,  124,  126,
+	  127,    0,   31,    9,   10,   13,   32,   58,   10,   13,   58,    9,
+	   10,   13,   32,    9,   10,   13,   32,    9,   10,   13,   32,    9,
+	   10,   13,   32,   58,    9,   10,   13,   32,    9,   10,   13,   32,
+	    9,   10,   13,   32,   10,   13,   13,   33,  124,  126,   35,   39,
+	   42,   43,   45,   46,   48,   57,   65,   90,   94,  122,   10,   33,
+	   58,  124,  126,   35,   39,   42,   43,   45,   46,   48,   57,   65,
+	   90,   94,  122,    9,   13,   32,  127,    0,   31,   10,    9,   13,
+	   32,   33,  124,  126,   35,   39,   42,   43,   45,   46,   48,   57,
+	   65,   90,   94,  122,    9,   13,   32,  127,    0,   31,    9,   13,
+	   32,  127,    0,   31,    9,   13,   32,  127,    0,   31,   10,   87,
+	   97,  122,   10,   97,  122,   10,   97,  122,   97,  122,   58,   97,
+	  122,   10,   32,   48,   49,   57,   32,   46,   48,   57,   48,   57,
+	   48,   57,   48,   57,   48,   57,   48,   57,   48,   57,   48,   57,
+	   48,   57,   32,   48,   57,   10,   32,   33,  124,  126,   35,   39,
+	   42,   43,   45,   46,   48,   57,   65,   90,   94,  122,   10,   32,
+	   48,   57,   10,   48,   57,   10,   48,   57,   10,   32,   48,   57,
+	   10,   32,   10,   32,   10,   32,   10,   32,   48,   57,   32,   48,
+	   57,   10,   32,   10,   32,   48,   57,   10,   48,   57,   10,   32,
+	   33,   47,  124,  126,   35,   39,   42,   43,   45,   57,   65,   90,
+	   94,  122,   10,   32,   33,  124,  126,   35,   39,   42,   43,   45,
+	   46,   48,   57,   65,   90,   94,  122,    9,   10,   32,   33,   59,
+	  124,  126,   35,   39,   42,   43,   45,   46,   48,   57,   65,   90,
+	   94,  122,    9,   10,   32,   59,    9,   32,   59,   48,   57,    9,
+	   32,   59,    9,   32,   33,  124,  126,   35,   39,   42,   43,   45,
+	   46,   48,   57,   65,   90,   94,  122,   33,   61,  124,  126,   35,
+	   39,   42,   43,   45,   46,   48,   57,   65,   90,   94,  122,   34,
+	  124,  126,   33,   39,   42,   43,   45,   46,   48,   57,   65,   90,
+	   94,  122,    9,   32,   33,   59,  124,  126,   35,   39,   42,   43,
+	   45,   46,   48,   57,   65,   90,   94,  122,    9,   34,   92,   32,
+	  126,  128,  255,    9,   32,   59,    0,  191,  194,  244,    9,   10,
+	   32,   33,  124,  126,   35,   39,   42,   43,   45,   46,   48,   57,
+	   65,   90,   94,  122,    9,   32,   33,  124,  126,   35,   39,   42,
+	   43,   45,   46,   48,   57,   65,   90,   94,  122,   10,   33,   61,
+	  124,  126,   35,   39,   42,   43,   45,   46,   48,   57,   65,   90,
+	   94,  122,   10,   33,   61,  124,  126,   35,   39,   42,   43,   45,
+	   46,   48,   57,   65,   90,   94,  122,   10,   32,   33,   61,  124,
+	  126,   35,   39,   42,   43,   45,   46,   48,   57,   65,   90,   94,
+	  122,   10,   33,   61,  124,  126,   35,   39,   42,   43,   45,   46,
+	   48,   57,   65,   90,   94,  122,   10,   32,   33,   61,  124,  126,
+	   35,   39,   42,   43,   45,   46,   48,   57,   65,   90,   94,  122,
+	   10,   32,   34,  124,  126,   33,   39,   42,   43,   45,   46,   48,
+	   57,   65,   90,   94,  122,    9,   10,   32,   34,   92,   33,  126,
+	  128,  255,    9,   34,   92,   32,   47,   48,   57,   58,  126,  128,
+	  255,    9,   10,   34,   92,   32,   47,   48,   57,   58,  126,  128,
+	  255,    9,   10,   34,   92,   32,   47,   48,   57,   58,  126,  128,
+	  255,    9,   10,   32,   34,   92,   33,   47,   48,   57,   58,  126,
+	  128,  255,    9,   10,   32,   34,   92,   33,  126,  128,  255,    9,
+	   10,   32,   34,   92,   33,  126,  128,  255,    9,   10,   32,   34,
+	   92,   33,  126,  128,  255,    9,   10,   32,   34,   92,   33,  126,
+	  128,  255,    9,   34,   92,   32,   47,   48,   57,   58,  126,  128,
+	  255,    9,   32,   34,   92,   33,   47,   48,   57,   58,  126,  128,
+	  255,    9,   10,   32,   34,   92,   33,  126,  128,  255,    9,   10,
+	   32,   34,   92,   33,  126,  128,  255,    9,   34,   92,   32,   47,
+	   48,   57,   58,  126,  128,  255,    9,   10,   34,   92,   32,   47,
+	   48,   57,   58,  126,  128,  255,    9,   10,   32,   59,    9,   10,
+	   32,   59,    9,   32,   59,   48,   57,    9,   10,   32,   33,  124,
+	  126,   35,   39,   42,   43,   45,   46,   48,   57,   65,   90,   94,
+	  122,    9,   32,   33,  124,  126,   35,   39,   42,   43,   45,   46,
+	   48,   57,   65,   90,   94,  122,   10,   32,   33,   61,  124,  126,
+	   35,   39,   42,   43,   45,   46,   48,   57,   65,   90,   94,  122,
+	   10,   32,   34,  124,  126,   33,   39,   42,   43,   45,   46,   48,
+	   57,   65,   90,   94,  122,    9,   10,   32,   33,   59,  124,  126,
 	   35,   39,   42,   43,   45,   46,   48,   57,   65,   90,   94,  122,
-	    9,   13,   32,  127,    0,   31,   10,    9,   13,   32,   33,  124,
+	   10,   32,    0,  191,  194,  244,    9,   10,   32,   59,    9,   10,
+	   32,   59,    9,   32,   59,   48,   57,    9,   10,   32,   33,  124,
 	  126,   35,   39,   42,   43,   45,   46,   48,   57,   65,   90,   94,
-	  122,    9,   13,   32,  127,    0,   31,    9,   13,   32,  127,    0,
-	   31,    9,   13,   32,  127,    0,   31,    0
+	  122,    9,   32,   33,  124,  126,   35,   39,   42,   43,   45,   46,
+	   48,   57,   65,   90,   94,  122,   32,   33,   61,  124,  126,   35,
+	   39,   42,   43,   45,   46,   48,   57,   65,   90,   94,  122,   10,
+	   32,   33,   61,  124,  126,   35,   39,   42,   43,   45,   46,   48,
+	   57,   65,   90,   94,  122,   10,   32,   34,  124,  126,   33,   39,
+	   42,   43,   45,   46,   48,   57,   65,   90,   94,  122,    9,   10,
+	   32,   33,   59,  124,  126,   35,   39,   42,   43,   45,   46,   48,
+	   57,   65,   90,   94,  122,    9,   32,   59,   48,   57,   10,   32,
+	   48,   57,   10,   32,   48,   57,   10,   32,   48,   57,   10,   32,
+	   10,   32,   10,   32,   48,   57,   10,   32,   48,   57,   10,   32,
+	   48,   57,   10,   32,    0,  191,  194,  244,    9,   10,   32,   59,
+	    9,   10,   32,   59,    9,   10,   32,   59,    9,   10,   32,   33,
+	  124,  126,   35,   39,   42,   43,   45,   46,   48,   57,   65,   90,
+	   94,  122,    9,   10,   32,   33,  124,  126,   35,   39,   42,   43,
+	   45,   46,   48,   57,   65,   90,   94,  122,   10,   32,   33,   61,
+	  124,  126,   35,   39,   42,   43,   45,   46,   48,   57,   65,   90,
+	   94,  122,   10,   32,   34,  124,  126,   33,   39,   42,   43,   45,
+	   46,   48,   57,   65,   90,   94,  122,    9,   10,   32,   33,   59,
+	  124,  126,   35,   39,   42,   43,   45,   46,   48,   57,   65,   90,
+	   94,  122,    9,   10,   32,   59,   48,   57,   10,   32,   48,   57,
+	   10,   32,   48,   57,   10,   32,   48,   57,   10,   32,   48,   57,
+	   10,   32,   48,   57,   10,   32,   10,   32,   48,   57,   10,   32,
+	    0,  191,  194,  244,   10,   32,    0,  191,  194,  244,   32,   48,
+	   57,   32,   48,   57,   32,   48,   57,   32,   48,   57,   32,   48,
+	   57,   32,   48,   57,   32,   48,   57,   32,   48,   57,   32,   48,
+	   57,   32,   48,   57,   32,   48,   57,   32,   48,   57,   32,   48,
+	   57,   32,   48,   57,   32,   48,   57,   32,   48,   57,   32,   48,
+	   57,   32,   48,   57,   32,   48,   57,   32,   48,   57,   46,   48,
+	   57,   48,   57,   46,   48,   57,   48,   57,   32,   48,   57,   32,
+	   48,   57,   32,   46,   48,   57,   46,   46,   48,   57,   46,   46,
+	   48,   57,   46,   46,   48,   57,   65,   82,   67,   47,   48,   57,
+	   46,   48,   57,   48,   57,   10,   13,   48,   57,    9,   10,   13,
+	   32,   58,   10,   13,   58,    9,   10,   13,   32,    9,   10,   13,
+	   32,    9,   10,   13,   32,    9,   10,   13,   32,   58,    9,   10,
+	   13,   32,    9,   10,   13,   32,    9,   10,   13,   32,   10,   13,
+	   10,   13,    0
 	};
 }
 
@@ -518,8 +656,23 @@ private static byte[] init__warc_single_lengths_0()
 	    1,    1,    1,    1,    1,    0,    1,    0,    1,    0,    1,    1,
 	    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
 	    0,    1,    0,    1,    1,    4,    1,    4,    4,    1,    6,    4,
-	    4,    4,    4,    1,    4,    4,    1,    6,    4,    4,    4,    0,
-	    0
+	    4,    4,    5,    3,    4,    4,    4,    5,    4,    4,    4,    2,
+	    4,    1,    4,    4,    1,    6,    4,    4,    4,    2,    1,    1,
+	    0,    1,    2,    1,    2,    0,    0,    0,    0,    0,    0,    0,
+	    0,    1,    5,    2,    0,    1,    1,    2,    2,    2,    2,    2,
+	    0,    1,    2,    2,    0,    1,    6,    5,    7,    4,    3,    3,
+	    5,    4,    3,    6,    3,    3,    0,    6,    5,    5,    5,    6,
+	    5,    6,    5,    5,    3,    4,    4,    5,    5,    5,    5,    5,
+	    3,    4,    5,    5,    3,    4,    4,    4,    3,    6,    5,    6,
+	    5,    7,    2,    4,    4,    3,    6,    5,    5,    6,    5,    7,
+	    3,    2,    2,    2,    2,    2,    2,    2,    2,    2,    4,    4,
+	    4,    6,    6,    6,    5,    7,    4,    2,    2,    2,    2,    2,
+	    2,    2,    0,    2,    2,    1,    1,    1,    1,    1,    1,    1,
+	    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+	    1,    0,    1,    0,    1,    0,    1,    1,    1,    1,    1,    1,
+	    1,    1,    1,    1,    1,    1,    1,    1,    0,    1,    0,    2,
+	    5,    3,    4,    4,    4,    5,    4,    4,    4,    2,    2,    0,
+	    0,    0,    0
 	};
 }
 
@@ -543,8 +696,23 @@ private static byte[] init__warc_range_lengths_0()
 	    1,    1,    1,    1,    0,    1,    1,    1,    1,    1,    1,    1,
 	    0,    1,    0,    1,    0,    1,    0,    1,    0,    0,    0,    0,
 	    1,    1,    1,    1,    0,    6,    0,    6,    1,    0,    6,    1,
-	    1,    1,    6,    0,    6,    1,    0,    6,    1,    1,    1,    0,
-	    0
+	    1,    1,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+	    6,    0,    6,    1,    0,    6,    1,    1,    1,    1,    1,    1,
+	    1,    1,    0,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+	    1,    1,    6,    0,    1,    1,    1,    1,    0,    0,    0,    0,
+	    1,    1,    0,    0,    1,    1,    5,    6,    6,    0,    1,    0,
+	    6,    6,    6,    6,    2,    0,    2,    6,    6,    6,    6,    6,
+	    6,    6,    6,    2,    4,    4,    4,    4,    2,    2,    2,    2,
+	    4,    4,    2,    2,    4,    4,    0,    0,    1,    6,    6,    6,
+	    6,    6,    2,    0,    0,    1,    6,    6,    6,    6,    6,    6,
+	    1,    1,    1,    1,    0,    0,    1,    1,    1,    2,    0,    0,
+	    0,    6,    6,    6,    6,    6,    1,    1,    1,    1,    1,    1,
+	    0,    0,    1,    2,    2,    1,    1,    1,    1,    1,    1,    1,
+	    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+	    0,    1,    1,    1,    1,    1,    1,    1,    0,    1,    0,    1,
+	    0,    1,    0,    1,    0,    0,    0,    0,    1,    1,    1,    1,
+	    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+	    0,    0,    0
 	};
 }
 
@@ -568,8 +736,23 @@ private static short[] init__warc_index_offsets_0()
 	  819,  822,  825,  828,  831,  833,  835,  838,  840,  843,  845,  848,
 	  851,  853,  856,  858,  861,  863,  866,  868,  871,  873,  875,  877,
 	  879,  881,  884,  886,  889,  891,  902,  904,  915,  921,  923,  936,
-	  942,  948,  954,  965,  967,  978,  984,  986,  999, 1005, 1011, 1017,
-	 1018
+	  942,  948,  954,  960,  964,  969,  974,  979,  985,  990,  995, 1000,
+	 1003, 1014, 1016, 1027, 1033, 1035, 1048, 1054, 1060, 1066, 1070, 1073,
+	 1076, 1078, 1081, 1084, 1087, 1091, 1093, 1095, 1097, 1099, 1101, 1103,
+	 1105, 1107, 1110, 1122, 1125, 1127, 1130, 1133, 1137, 1140, 1143, 1146,
+	 1149, 1151, 1154, 1157, 1160, 1162, 1165, 1177, 1189, 1203, 1208, 1213,
+	 1217, 1229, 1240, 1250, 1263, 1269, 1273, 1276, 1289, 1301, 1313, 1325,
+	 1338, 1350, 1363, 1375, 1383, 1391, 1400, 1409, 1419, 1427, 1435, 1443,
+	 1451, 1459, 1468, 1476, 1484, 1492, 1501, 1506, 1511, 1516, 1529, 1541,
+	 1554, 1566, 1580, 1585, 1590, 1595, 1600, 1613, 1625, 1637, 1650, 1662,
+	 1676, 1681, 1685, 1689, 1693, 1696, 1699, 1703, 1707, 1711, 1716, 1721,
+	 1726, 1731, 1744, 1757, 1770, 1782, 1796, 1802, 1806, 1810, 1814, 1818,
+	 1822, 1825, 1828, 1830, 1835, 1840, 1843, 1846, 1849, 1852, 1855, 1858,
+	 1861, 1864, 1867, 1870, 1873, 1876, 1879, 1882, 1885, 1888, 1891, 1894,
+	 1897, 1899, 1901, 1904, 1906, 1909, 1911, 1914, 1917, 1919, 1922, 1924,
+	 1927, 1929, 1932, 1934, 1937, 1939, 1941, 1943, 1945, 1947, 1950, 1952,
+	 1956, 1962, 1966, 1971, 1976, 1981, 1987, 1992, 1997, 2002, 2005, 2008,
+	 2009, 2010, 2011
 	};
 }
 
@@ -658,12 +841,95 @@ private static short[] init__warc_indicies_0()
 	  163,  163,    1,  166,  167,  166,    1,    1,  168,  169,    1,  170,
 	  171,  170,  172,  172,  172,  172,  172,  172,  172,  172,  172,    1,
 	  170,  173,  170,    1,    1,  174,  175,  176,  175,    1,    1,  168,
-	  177,  167,  177,    1,    1,  168,  178,  179,  179,  179,  179,  179,
-	  179,  179,  179,  179,    1,  180,    1,  179,  181,  179,  179,  179,
-	  179,  179,  179,  179,  179,    1,  182,  183,  182,    1,    1,  184,
-	  185,    1,  186,  187,  186,  188,  188,  188,  188,  188,  188,  188,
-	  188,  188,    1,  186,  189,  186,    1,    1,  190,  191,  192,  191,
-	    1,    1,  184,  193,  183,  193,    1,    1,  184,    1,    1,    0
+	  177,  167,  177,    1,    1,  168,    1,  179,  180,    1,    1,  178,
+	    1,    1,  181,  178,  183,  184,  185,  183,  182,  186,  187,  188,
+	  186,  182,  189,  184,  185,  189,  182,  191,  192,  193,  191,    1,
+	  190,  191,  195,  196,  191,  194,  197,  184,  185,  197,  182,  197,
+	    1,  197,  197,  182,  179,  180,    1,  198,  199,  199,  199,  199,
+	  199,  199,  199,  199,  199,    1,  200,    1,  199,  201,  199,  199,
+	  199,  199,  199,  199,  199,  199,    1,  202,  203,  202,    1,    1,
+	  204,  205,    1,  206,  207,  206,  208,  208,  208,  208,  208,  208,
+	  208,  208,  208,    1,  206,  209,  206,    1,    1,  210,  211,  212,
+	  211,    1,    1,  204,  213,  203,  213,    1,    1,  204,  214,  215,
+	  216,    1,  217,  216,    1,  218,  216,    1,  216,    1,  219,  216,
+	    1,    1,  220,  219,  221,  222,    1,  223,  224,  225,    1,  226,
+	    1,  227,    1,  228,    1,  229,    1,  230,    1,  231,    1,  232,
+	    1,  233,    1,  234,  235,    1,    1,  237,  238,  238,  238,  238,
+	  238,  238,  238,  238,  238,  236,    1,  237,  236,  239,    1,  240,
+	  241,    1,  240,  242,    1,  240,  243,  244,    1,    1,    1,  245,
+	    1,  246,  245,    1,    1,  247,    1,  248,  247,  249,    1,  250,
+	  249,    1,    1,    1,  251,    1,  252,  251,  244,    1,  240,  244,
+	    1,    1,  237,  238,  253,  238,  238,  238,  238,  238,  238,  238,
+	  236,    1,  237,  254,  254,  254,  254,  254,  254,  254,  254,  254,
+	  236,  255,    1,  256,  254,  257,  254,  254,  254,  254,  254,  254,
+	  254,  254,  236,  255,    1,  256,  257,  236,  258,  258,  259,  239,
+	    1,  258,  258,  259,    1,  259,  259,  260,  260,  260,  260,  260,
+	  260,  260,  260,  260,    1,  260,  261,  260,  260,  260,  260,  260,
+	  260,  260,  260,    1,  263,  262,  262,  262,  262,  262,  262,  262,
+	  262,    1,  258,  256,  262,  259,  262,  262,  262,  262,  262,  262,
+	  262,  262,    1,  263,  264,  265,  263,  263,    1,  258,  256,  259,
+	    1,  263,  263,    1,  257,    1,  266,  267,  267,  267,  267,  267,
+	  267,  267,  267,  267,  236,  259,  259,  260,  260,  260,  260,  260,
+	  260,  268,  260,  260,    1,  240,  260,  261,  260,  260,  260,  260,
+	  260,  269,  260,  260,    1,  240,  260,  261,  260,  260,  260,  260,
+	  260,  270,  260,  260,    1,  240,  243,  260,  261,  260,  260,  260,
+	  260,  260,  271,  260,  260,    1,  240,  260,  261,  260,  260,  260,
+	  260,  260,  271,  260,  260,    1,    1,  237,  267,  272,  267,  267,
+	  267,  267,  267,  267,  267,  267,  236,    1,  237,  273,  254,  254,
+	  254,  254,  254,  254,  254,  254,  236,  273,    1,  274,  255,  275,
+	  273,  273,  236,  263,  264,  265,  263,  276,  263,  263,    1,  263,
+	  240,  264,  265,  263,  277,  263,  263,    1,  263,  240,  264,  265,
+	  263,  278,  263,  263,    1,  263,  240,  279,  264,  265,  263,  280,
+	  263,  263,    1,  281,    1,  263,  282,  283,  281,  281,  245,  281,
+	    1,  284,  282,  283,  281,  281,  245,  285,    1,  263,  286,  287,
+	  285,  285,  247,  285,    1,  288,  286,  287,  285,  285,  247,  263,
+	  264,  265,  263,  289,  263,  263,    1,  263,  290,  264,  265,  263,
+	  289,  263,  263,    1,  291,    1,  263,  292,  293,  291,  291,  251,
+	  291,    1,  294,  292,  293,  291,  291,  251,  263,  264,  265,  263,
+	  280,  263,  263,    1,  263,  240,  264,  265,  263,  280,  263,  263,
+	    1,  295,    1,  256,  296,  251,  295,    1,  297,  296,  251,  258,
+	  258,  259,  244,    1,  296,    1,  298,  299,  299,  299,  299,  299,
+	  299,  299,  299,  299,  251,  259,  259,  260,  260,  260,  260,  260,
+	  260,  271,  260,  260,    1,    1,  252,  299,  300,  299,  299,  299,
+	  299,  299,  299,  299,  299,  251,    1,  252,  291,  301,  301,  301,
+	  301,  301,  301,  301,  301,  251,  295,    1,  256,  301,  296,  301,
+	  301,  301,  301,  301,  301,  301,  301,  251,  263,  294,  291,  291,
+	  251,  302,    1,  303,  304,  247,  302,    1,  305,  304,  247,  258,
+	  258,  259,  249,    1,  304,    1,  306,  307,  307,  307,  307,  307,
+	  307,  307,  307,  307,  247,  259,  259,  260,  260,  260,  260,  260,
+	  260,  308,  260,  260,    1,  250,  260,  261,  260,  260,  260,  260,
+	  260,  308,  260,  260,    1,    1,  248,  307,  309,  307,  307,  307,
+	  307,  307,  307,  307,  307,  247,    1,  248,  285,  310,  310,  310,
+	  310,  310,  310,  310,  310,  247,  302,    1,  303,  310,  304,  310,
+	  310,  310,  310,  310,  310,  310,  310,  247,  258,  258,  259,  311,
+	    1,  240,  250,  312,    1,  240,  250,  313,    1,  240,  314,  315,
+	    1,    1,    1,  316,    1,  317,  316,    1,    1,  318,  247,  240,
+	  248,  318,  247,  240,  250,  315,    1,  263,  288,  285,  285,  247,
+	  319,    1,  320,  321,  245,  319,    1,  322,  321,  245,  302,    1,
+	  258,  304,  247,  321,    1,  323,  324,  324,  324,  324,  324,  324,
+	  324,  324,  324,  245,  304,    1,  259,  307,  307,  307,  307,  307,
+	  307,  307,  307,  307,  247,    1,  246,  324,  325,  324,  324,  324,
+	  324,  324,  324,  324,  324,  245,    1,  246,  281,  326,  326,  326,
+	  326,  326,  326,  326,  326,  245,  319,    1,  320,  326,  321,  326,
+	  326,  326,  326,  326,  326,  326,  326,  245,  302,    1,  258,  304,
+	  327,  247,  240,  248,  328,  247,  240,  248,  329,  247,  240,  330,
+	  318,  247,    1,    1,  331,  245,    1,  332,  331,  245,    1,    1,
+	  333,    1,  334,  333,  315,    1,  263,  284,  281,  281,  245,  263,
+	  274,  273,  273,  236,  234,  335,    1,  234,  336,    1,  234,  337,
+	    1,  234,  338,    1,  234,  339,    1,  234,  340,    1,  234,  341,
+	    1,  234,  342,    1,  234,  343,    1,  234,  344,    1,  234,  345,
+	    1,  234,  346,    1,  234,  347,    1,  234,  348,    1,  234,  349,
+	    1,  234,  350,    1,  234,  351,    1,  234,  352,    1,  234,  353,
+	    1,  234,    1,  354,    1,  355,  356,    1,  357,    1,  358,  359,
+	    1,  360,    1,  223,  361,    1,  223,  362,    1,  223,    1,  358,
+	  363,    1,  358,    1,  355,  364,    1,  355,    1,  224,  365,    1,
+	  224,    1,  224,  225,    1,  366,    1,  367,    1,  368,    1,  369,
+	    1,  370,    1,  371,  370,    1,  372,    1,  373,  374,  372,    1,
+	    1,  376,  377,    1,    1,  375,    1,    1,  378,  375,  380,  381,
+	  382,  380,  379,  383,  384,  385,  383,  379,  386,  381,  382,  386,
+	  379,  388,  389,  390,  388,    1,  387,  388,  392,  393,  388,  391,
+	  394,  381,  382,  394,  379,  394,    1,  394,  394,  379,  376,  377,
+	    1,  373,  374,    1,    1,    1,    1,    1,    0
 	};
 }
 
@@ -675,7 +941,7 @@ private static short[] init__warc_trans_targs_0()
 	return new short [] {
 	    2,    0,  152,    5,    3,    4,    6,    7,    8,  151,    9,  137,
 	  149,   10,   11,   12,   13,   14,   15,   16,   17,   18,  117,   19,
-	   20,   34,   21,  179,   22,   23,   24,   33,   25,   26,   27,   28,
+	   20,   34,   21,  359,   22,   23,   24,   33,   25,   26,   27,   28,
 	   29,   30,   31,   32,   35,   36,   37,   38,   47,   39,   40,   41,
 	   42,   43,   44,   45,   46,   48,   53,   49,   50,   51,   52,   54,
 	   55,   56,  116,   57,   58,   59,   60,   69,   61,   98,  115,   62,
@@ -686,10 +952,26 @@ private static short[] init__warc_trans_targs_0()
 	  113,  114,  118,  119,  120,  121,  122,  123,  124,  125,  126,  127,
 	  128,  129,  130,  131,  132,  133,  134,  135,  136,  138,  139,  147,
 	  140,  141,  145,  142,  143,  144,  146,  148,  150,  153,  154,  155,
-	  156,  157,  158,  159,  160,  161,  162,  163,  179,  164,  164,  165,
-	  168,  166,  167,  162,  163,  165,  168,  169,  165,  169,  171,  172,
-	  180,  173,  173,  174,  177,  175,  176,  171,  172,  174,  177,  178,
-	  174,  178
+	  156,  157,  158,  159,  160,  161,  162,  163,  359,  164,  164,  165,
+	  168,  166,  167,  162,  163,  165,  168,  169,  165,  169,  171,  360,
+	  179,  172,  173,  172,  175,  177,  174,  175,  177,  174,  171,  176,
+	  360,  179,  173,  175,  177,  178,  181,  182,  361,  183,  183,  184,
+	  187,  185,  186,  181,  182,  184,  187,  188,  184,  188,  190,  340,
+	  193,  191,  192,  194,  195,  196,  339,  197,  325,  337,  198,  199,
+	  200,  201,  202,  203,  204,  205,  206,  305,  207,  208,  222,  209,
+	  362,  210,  211,  212,  221,  213,  214,  215,  216,  217,  218,  219,
+	  220,  223,  224,  225,  226,  235,  227,  228,  229,  230,  231,  232,
+	  233,  234,  236,  241,  237,  238,  239,  240,  242,  243,  244,  304,
+	  245,  246,  247,  248,  257,  249,  286,  303,  250,  251,  267,  285,
+	  252,  253,  254,  255,  258,  266,  256,  259,  261,  260,  262,  263,
+	  264,  265,  268,  276,  270,  269,  271,  273,  272,  274,  275,  277,
+	  278,  279,  280,  284,  281,  282,  283,  287,  294,  289,  288,  290,
+	  291,  292,  293,  295,  296,  297,  298,  299,  300,  301,  302,  306,
+	  307,  308,  309,  310,  311,  312,  313,  314,  315,  316,  317,  318,
+	  319,  320,  321,  322,  323,  324,  326,  327,  335,  328,  329,  333,
+	  330,  331,  332,  334,  336,  338,  341,  342,  343,  344,  345,  346,
+	  347,  348,  358,  349,  362,  357,  350,  351,  350,  353,  355,  352,
+	  353,  355,  352,  349,  354,  362,  357,  351,  353,  355,  356
 	};
 }
 
@@ -701,7 +983,7 @@ private static byte[] init__warc_trans_actions_0()
 	return new byte [] {
 	    0,    0,    0,    1,    0,    0,    1,   15,    1,    1,   17,    1,
 	    1,    1,    1,    1,    1,    1,    1,    1,    1,   19,    1,    0,
-	    0,    0,    1,   34,    1,    1,   21,    1,    0,    0,    0,    0,
+	    0,    0,    1,   43,    1,    1,   21,    1,    0,    0,    0,    0,
 	    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
 	    0,    0,    0,    0,    0,    0,    0,    1,    1,    1,    1,    0,
 	    0,    0,    0,    1,    1,    1,   21,    1,    0,    0,    0,    0,
@@ -712,10 +994,26 @@ private static byte[] init__warc_trans_actions_0()
 	    0,    0,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
 	    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
 	    1,    1,    1,    1,    1,    1,    1,    1,    1,    0,    0,    0,
-	    0,    3,    0,    5,    0,    0,    0,    1,   23,   11,    0,    0,
-	    1,    0,    0,   13,   31,    9,   28,   25,    7,    1,    0,    1,
-	    0,   11,    0,    0,    1,    0,    0,   13,   31,    9,   28,   25,
-	    7,    1
+	    0,    3,    0,    5,    0,    0,    0,    1,   25,   11,    0,    0,
+	    1,    0,    0,   13,   33,    9,   30,   27,    7,    1,    1,    0,
+	    0,   11,    1,    0,    0,    1,   27,    7,   27,    1,   33,    0,
+	   13,   13,   30,    9,   30,    1,    0,    1,    0,   11,    0,    0,
+	    1,    0,    0,   13,   33,    9,   30,   27,    7,    1,    0,    0,
+	    1,    0,    0,    1,   15,    1,    1,   17,    1,    1,    1,    1,
+	    1,    1,    1,    1,    1,    1,   19,    1,    0,    0,    0,    1,
+	   39,    1,    1,   21,    1,    0,    0,    0,    0,    0,    0,    0,
+	    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+	    0,    0,    0,    0,    1,    1,    1,    1,    0,    0,    0,    0,
+	    1,    1,    1,   21,    1,    0,    0,    0,    0,    0,    0,    0,
+	    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+	    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    1,
+	    1,    1,   21,    1,    0,    0,    1,    0,    0,    0,    0,    0,
+	    0,    0,    0,    1,    1,    1,   21,    0,    0,    0,    0,    1,
+	    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+	    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+	    1,    1,    1,    1,    1,    1,    0,    0,    0,    0,    3,    0,
+	    5,    0,    0,    1,   23,    0,   11,    1,    0,    0,    1,   27,
+	    7,   27,    1,   33,    0,   36,   13,   30,    9,   30,    1
 	};
 }
 
@@ -723,12 +1021,14 @@ private static byte[] init__warc_trans_actions_0()
 
 
 static final int warc_start = 1;
-static final int warc_first_final = 179;
+static final int warc_first_final = 359;
 static final int warc_error = 0;
 
-static final int warc_en_warc_fields = 170;
+static final int warc_en_warc_fields_lenient = 170;
+static final int warc_en_warc_fields = 180;
+static final int warc_en_any_header_lenient = 189;
 static final int warc_en_any_header = 1;
 
 
-// line 284 "WarcParser.rl"
+// line 324 "WarcParser.rl"
 }
\ No newline at end of file
diff --git a/src/org/netpreserve/jwarc/WarcParser.rl b/src/org/netpreserve/jwarc/WarcParser.rl
index 20c16e1..db6e781 100644
--- a/src/org/netpreserve/jwarc/WarcParser.rl
+++ b/src/org/netpreserve/jwarc/WarcParser.rl
@@ -137,6 +137,20 @@ named_field = field_name ":" field_value CRLF %handle_value;
 named_fields = named_field* CRLF;
 warc_header = version named_fields;
 
+CRLF_lenient = "\r"* "\n";
+LWS_lenient = CRLF_lenient RWS;
+TEXT_lenient = ((any - '\n' - WS) (any - '\n')*)? (any - '\n' - WS - '\r') %end_of_text;
+
+version_lenient = "WARC/" version_major "." version_minor+ CRLF_lenient ;
+
+field_name_lenient = ((any - '\r' - '\n' - ' ' - '\t' - ':') (any - '\r' - '\n' - ':')*) $push %handle_name;
+field_value_first_lenient = OWS (TEXT_lenient OWS)? $push;
+field_value_folded_lenient = LWS_lenient (TEXT_lenient OWS)? >fold $push;
+field_value_lenient = field_value_first_lenient (field_value_folded_lenient)*;
+named_field_lenient = field_name_lenient ":" field_value_lenient CRLF_lenient %handle_value;
+named_fields_lenient = named_field_lenient* CRLF_lenient;
+warc_header_lenient = version_lenient named_fields_lenient;
+
 token = (ascii - CTL - separators)+;
 obs_text = 0x80..0xff;
 qdtext = "\t" | " " | 0x21 | 0x23..0x5b | 0x5d..0x7e | obs_text;
@@ -163,7 +177,9 @@ arc_v2_fields = arc_v2_status " " arc_v2_checksum " " arc_v2_location " " arc_v2
 arc_header = "\n"{0,3} arc_url " " arc_ip " " arc_date_lenient " " arc_mime_lenient
              " " (arc_v2_fields " ")? arc_length "\n";
 
+warc_fields_lenient := named_fields_lenient;
 warc_fields := named_fields;
+any_header_lenient := (arc_header | warc_header_lenient) @{ fbreak; };
 any_header := (arc_header | warc_header) @{ fbreak; };
 
 }%%
@@ -215,6 +231,31 @@ public class WarcParser extends MessageParser {
         }
     }
 
+	/**
+	 * Sets the lenient mode for the WarcParser.
+	 * <p>
+	 * When enabled, this causes the parser to follow the specification less strictly,
+	 * allowing reading of non-compliant records by:
+	 * <ul>
+	 *   <li>permitting ASCII control characters in header field names and values
+	 *   <li>allowing lines to end with LF instead of CRLF
+	 *   <li>permitting multi-digit WARC minor versions like "0.18"
+	 * </ul>
+	 * Calling this method also resets the state of the parser.
+	 */
+    public void setLenient(boolean lenient) {
+        if (warcFieldsMode()) {
+            entryState = lenient ? warc_en_warc_fields_lenient : warc_en_warc_fields;
+        } else {
+            entryState = lenient ? warc_en_any_header_lenient : warc_start;
+        }
+        reset();
+    }
+
+    private boolean warcFieldsMode() {
+        return entryState == warc_en_warc_fields || entryState == warc_en_warc_fields_lenient;
+    }
+
     public boolean isFinished() {
         return cs >= warc_first_final;
     }
diff --git a/src/org/netpreserve/jwarc/WarcReader.java b/src/org/netpreserve/jwarc/WarcReader.java
index fb230bd..8c0c1fc 100644
--- a/src/org/netpreserve/jwarc/WarcReader.java
+++ b/src/org/netpreserve/jwarc/WarcReader.java
@@ -365,6 +365,21 @@ public void onWarning(Consumer<String> warningHandler) {
         parser.onWarning(warningHandler);
     }
 
+    /**
+     * Sets the lenient mode for the WarcParser.
+     * <p>
+     * When enabled, this causes the parser to follow the specification less strictly,
+     * allowing reading of non-compliant records by:
+     * <ul>
+     *   <li>permitting ASCII control characters in header field names and values
+     *   <li>allowing lines to end with LF instead of CRLF
+     *   <li>permitting multi-digit WARC minor versions like "0.18"
+     * </ul>
+     */
+    public void setLenient(boolean lenient) {
+        parser.setLenient(lenient);
+    }
+
     /**
      * Closes the underlying channel.
      */
diff --git a/src/org/netpreserve/jwarc/cdx/CdxWriter.java b/src/org/netpreserve/jwarc/cdx/CdxWriter.java
index 741894f..fc48aff 100644
--- a/src/org/netpreserve/jwarc/cdx/CdxWriter.java
+++ b/src/org/netpreserve/jwarc/cdx/CdxWriter.java
@@ -68,6 +68,7 @@ public void write(WarcCaptureRecord capture, String filename,
     public void process(List<Path> warcFiles, boolean useAbsolutePaths) throws IOException {
         for (Path file : warcFiles) {
             try (WarcReader reader = new WarcReader(file)) {
+                reader.setLenient(true);
                 String filename = (useAbsolutePaths ? file.toAbsolutePath() : file.getFileName()).toString();
                 reader.onWarning(message -> emitWarning(filename, reader.position(), message));
                 process(reader, filename);
diff --git a/test/org/netpreserve/jwarc/WarcParserTest.java b/test/org/netpreserve/jwarc/WarcParserTest.java
index dec7381..9be0949 100644
--- a/test/org/netpreserve/jwarc/WarcParserTest.java
+++ b/test/org/netpreserve/jwarc/WarcParserTest.java
@@ -31,8 +31,24 @@ public void testParsingArcWithCorruptDates() {
         assertEquals(Optional.empty(), parser.headers().first("WARC-Date"));
     }
 
+    @Test
+    public void testLenientParsing() {
+        WarcParser parser = parse( "WARC/0.18\nHello\u0007:\u0008world\r\n\r\n", true);
+        assertEquals(Optional.of("\u0008world"), parser.headers().sole("Hello\u0007"));
+    }
+
+    @Test(expected = AssertionError.class)
+    public void testStrictParsing() {
+        parse( "WARC/1.0\r\nHello\u0007:\u0008world\r\n\r\n");
+    }
+
     private static WarcParser parse(String input) {
+        return parse(input, false);
+    }
+
+    private static WarcParser parse(String input, boolean lenient) {
         WarcParser parser = new WarcParser();
+        parser.setLenient(lenient);
         parser.parse(ByteBuffer.wrap(input.getBytes(StandardCharsets.ISO_8859_1)));
         assertFalse(parser.isError());
         assertTrue(parser.isFinished());