Fixed MimeParser/MimeMessage/HeaderList to deal with invalid headers …

…better

Partial fix for issue #499

MimeKit/Cryptography/DkimVerifierBase.cs

@@ -493,7 +493,7 @@ internal static Header GetSignedSignatureHeader (Header header)
 
 			Array.Resize (ref rawValue, length);
 
-			return new Header (header.Options, header.RawField, rawValue);
+			return new Header (header.Options, header.RawField, rawValue, false);
 		}
 	}
 }

MimeKit/Header.cs

@@ -291,12 +291,12 @@ internal Header (ParserOptions options, HeaderId id, string name, byte[] field,
 			Id = id;
 		}
 
-		internal Header (ParserOptions options, byte[] field, byte[] value)
+		internal Header (ParserOptions options, byte[] field, byte[] value, bool invalid)
 		{
 			var chars = new char[field.Length];
 			int count = 0;
 
-			while (count < field.Length && !field[count].IsBlank ()) {
+			while (count < field.Length && (invalid || !field[count].IsBlank ())) {
 				chars[count] = (char) field[count];
 				count++;
 			}
@@ -307,6 +307,7 @@ internal Header (ParserOptions options, byte[] field, byte[] value)
 
 			Field = new string (chars, 0, count);
 			Id = Field.ToHeaderId ();
+			IsInvalid = invalid;
 		}
 
 		internal Header (ParserOptions options, HeaderId id, string field, byte[] value)
@@ -327,7 +328,9 @@ internal Header (ParserOptions options, HeaderId id, string field, byte[] value)
 		/// <returns>A copy of the header with its current state.</returns>
 		public Header Clone ()
 		{
-			var header = new Header (Options, Id, Field, RawField, RawValue);
+			var header = new Header (Options, Id, Field, rawField, rawValue) {
+				IsInvalid = IsInvalid
+			};
 
 			// if the textValue has already been calculated, set it on the cloned header as well.
 			header.textValue = textValue;
@@ -369,6 +372,10 @@ public HeaderId Id {
 			get; private set;
 		}
 
+		internal bool IsInvalid {
+			get; private set;
+		}
+
 		/// <summary>
 		/// Gets the raw field name of the header.
 		/// </summary>
@@ -1177,7 +1184,7 @@ void OnChanged ()
 		/// <returns>A string representing the <see cref="Header"/>.</returns>
 		public override string ToString ()
 		{
-			return Field + ": " + Value;
+			return IsInvalid ? Field : Field + ": " + Value;
 		}
 
 		/// <summary>
@@ -1250,6 +1257,7 @@ internal static unsafe bool TryParse (ParserOptions options, byte* input, int le
 			byte* inend = input + length;
 			byte* start = input;
 			byte* inptr = input;
+			var invalid = false;
 
 			// find the end of the field name
 			if (strict) {
@@ -1264,8 +1272,13 @@ internal static unsafe bool TryParse (ParserOptions options, byte* input, int le
 				inptr++;
 
 			if (inptr == inend || *inptr != ':') {
-				header = null;
-				return false;
+				if (strict) {
+					header = null;
+					return false;
+				}
+
+				invalid = true;
+				inptr = inend;
 			}
 
 			var field = new byte[(int) (inptr - start)];
@@ -1276,19 +1289,25 @@ internal static unsafe bool TryParse (ParserOptions options, byte* input, int le
 					*outptr++ = *start++;
 			}
 
-			inptr++;
+			byte[] value;
 
-			int count = (int) (inend - inptr);
-			var value = new byte[count];
+			if (inptr < inend) {
+				inptr++;
 
-			fixed (byte *outbuf = value) {
-				byte* outptr = outbuf;
+				int count = (int) (inend - inptr);
+				value = new byte[count];
+
+				fixed (byte* outbuf = value) {
+					byte* outptr = outbuf;
 
-				while (inptr < inend)
-					*outptr++ = *inptr++;
+					while (inptr < inend)
+						*outptr++ = *inptr++;
+				}
+			} else {
+				value = new byte[0];
 			}
 
-			header = new Header (options, field, value);
+			header = new Header (options, field, value, invalid);
 
 			return true;
 		}

MimeKit/HeaderList.cs

@@ -699,11 +699,14 @@ public string this [string field] {
 				filtered.Add (options.CreateNewLineFilter ());
 
 				foreach (var header in headers) {
-					var rawValue = header.GetRawValue (options);
-
 					filtered.Write (header.RawField, 0, header.RawField.Length, cancellationToken);
-					filtered.Write (Header.Colon, 0, Header.Colon.Length, cancellationToken);
-					filtered.Write (rawValue, 0, rawValue.Length, cancellationToken);
+
+					if (!header.IsInvalid) {
+						var rawValue = header.GetRawValue (options);
+
+						filtered.Write (Header.Colon, 0, Header.Colon.Length, cancellationToken);
+						filtered.Write (rawValue, 0, rawValue.Length, cancellationToken);
+					}
 				}
 
 				filtered.Flush (cancellationToken);
@@ -752,11 +755,14 @@ public string this [string field] {
 				filtered.Add (options.CreateNewLineFilter ());
 
 				foreach (var header in headers) {
-					var rawValue = header.GetRawValue (options);
-
 					await filtered.WriteAsync (header.RawField, 0, header.RawField.Length, cancellationToken).ConfigureAwait (false);
-					await filtered.WriteAsync (Header.Colon, 0, Header.Colon.Length, cancellationToken).ConfigureAwait (false);
-					await filtered.WriteAsync (rawValue, 0, rawValue.Length, cancellationToken).ConfigureAwait (false);
+
+					if (!header.IsInvalid) {
+						var rawValue = header.GetRawValue (options);
+
+						await filtered.WriteAsync (Header.Colon, 0, Header.Colon.Length, cancellationToken).ConfigureAwait (false);
+						await filtered.WriteAsync (rawValue, 0, rawValue.Length, cancellationToken).ConfigureAwait (false);
+					}
 				}
 
 				await filtered.FlushAsync (cancellationToken).ConfigureAwait (false);

MimeKit/MimeMessage.cs

@@ -1094,11 +1094,14 @@ public virtual void Prepare (EncodingConstraint constraint, int maxLineLength =
 						if (options.HiddenHeaders.Contains (header.Id))
 							continue;
 
-						var rawValue = header.GetRawValue (options);
-
 						filtered.Write (header.RawField, 0, header.RawField.Length, cancellationToken);
-						filtered.Write (Header.Colon, 0, Header.Colon.Length, cancellationToken);
-						filtered.Write (rawValue, 0, rawValue.Length, cancellationToken);
+
+						if (!header.IsInvalid) {
+							var rawValue = header.GetRawValue (options);
+
+							filtered.Write (Header.Colon, 0, Header.Colon.Length, cancellationToken);
+							filtered.Write (rawValue, 0, rawValue.Length, cancellationToken);
+						}
 					}
 
 					filtered.Flush (cancellationToken);
@@ -1167,11 +1170,14 @@ public virtual void Prepare (EncodingConstraint constraint, int maxLineLength =
 						if (options.HiddenHeaders.Contains (header.Id))
 							continue;
 
-						var rawValue = header.GetRawValue (options);
-
 						await filtered.WriteAsync (header.RawField, 0, header.RawField.Length, cancellationToken).ConfigureAwait (false);
-						await filtered.WriteAsync (Header.Colon, 0, Header.Colon.Length, cancellationToken).ConfigureAwait (false);
-						await filtered.WriteAsync (rawValue, 0, rawValue.Length, cancellationToken).ConfigureAwait (false);
+
+						if (!header.IsInvalid) {
+							var rawValue = header.GetRawValue (options);
+
+							await filtered.WriteAsync (Header.Colon, 0, Header.Colon.Length, cancellationToken).ConfigureAwait (false);
+							await filtered.WriteAsync (rawValue, 0, rawValue.Length, cancellationToken).ConfigureAwait (false);
+						}
 					}
 
 					await filtered.FlushAsync (cancellationToken).ConfigureAwait (false);

MimeKit/MimeParser.cs

@@ -681,14 +681,7 @@ unsafe void ParseAndAppendHeader ()
 			fixed (byte* buf = headerBuffer) {
 				Header header;
 
-				if (!Header.TryParse (options, buf, headerIndex, false, out header)) {
-#if DEBUG
-					Debug.WriteLine (string.Format ("Invalid header at offset {0}: {1}", headerOffset, ConvertToCString (headerBuffer, 0, headerIndex)));
-#endif
-					headerIndex = 0;
-					return;
-				}
-
+				Header.TryParse (options, buf, headerIndex, false, out header);
 				header.Offset = headerOffset;
 				headers.Add (header);
 				headerIndex = 0;

UnitTests/MimeMessageTests.cs

@@ -538,6 +538,75 @@ This is the preamble.
 			}
 		}
 
+		[Test]
+		public async Task TestReserializationInvalidHeaders ()
+		{
+			string rawMessageText = @"From: Example Test <test@example.com>
+MIME-Version: 1.0
+Content-Type: multipart/mixed;
+   boundary=""simple boundary""
+Example: test
+Test
+Test Test
+Test:
+Test: 
+Test: Test
+Test Example:
+
+This is the preamble.
+
+--simple boundary
+Content-TypeS: text/plain
+
+This is a test.
+
+--simple boundary
+Content-Type: text/plain;
+Content-Disposition: attachment;
+Content-Transfer-Encoding: test;
+Content-Transfer-Encoding: binary;
+Test Test Test: Test Test
+Te$t($)*$= Test Test: Abc def
+test test = test
+test test :: test
+filename=""test.txt""
+
+Another test.
+
+--simple boundary--
+
+
+This is the epilogue.
+".Replace ("\r\n", "\n");
+
+			using (var source = new MemoryStream (Encoding.UTF8.GetBytes (rawMessageText))) {
+				var parser = new MimeParser (source, MimeFormat.Default);
+				var message = parser.ParseMessage ();
+
+				using (var serialized = new MemoryStream ()) {
+					var options = FormatOptions.Default.Clone ();
+					options.NewLineFormat = NewLineFormat.Unix;
+
+					message.WriteTo (options, serialized);
+
+					var result = Encoding.UTF8.GetString (serialized.ToArray ());
+
+					Assert.AreEqual (rawMessageText, result, "Reserialized message is not identical to the original.");
+				}
+
+				using (var serialized = new MemoryStream ()) {
+					var options = FormatOptions.Default.Clone ();
+					options.NewLineFormat = NewLineFormat.Unix;
+
+					await message.WriteToAsync (options, serialized);
+
+					var result = Encoding.UTF8.GetString (serialized.ToArray ());
+
+					Assert.AreEqual (rawMessageText, result, "Reserialized (async) message is not identical to the original.");
+				}
+			}
+		}
+
 		[Test]
 		public void TestMailMessageToMimeMessage ()
 		{