From 9f0222a561c6090c32fa78256449476fe21db86c Mon Sep 17 00:00:00 2001 From: Ivan Yurchenko Date: Wed, 10 Jun 2020 13:28:19 +0300 Subject: [PATCH] Encode hashes hexadecimally instead of Base64 https://github.com/aiven/aiven-kafka-connect-transforms/issues/16 Also: - In `HashTest.java`, use explicit expected hashed values. - In `HashTest.java`, adds a test to check same input produces same output continuously. --- .../aiven/kafka/connect/transforms/Hash.java | 6 +- .../kafka/connect/transforms/utils/Hex.java | 35 + .../kafka/connect/transforms/HashTest.java | 86 +- .../connect/transforms/utils/HexTest.java | 68 ++ src/test/resources/blns.txt | 744 ++++++++++++++++++ 5 files changed, 915 insertions(+), 24 deletions(-) create mode 100644 src/main/java/io/aiven/kafka/connect/transforms/utils/Hex.java create mode 100644 src/test/java/io/aiven/kafka/connect/transforms/utils/HexTest.java create mode 100644 src/test/resources/blns.txt diff --git a/src/main/java/io/aiven/kafka/connect/transforms/Hash.java b/src/main/java/io/aiven/kafka/connect/transforms/Hash.java index 33f4d38..449bac4 100644 --- a/src/main/java/io/aiven/kafka/connect/transforms/Hash.java +++ b/src/main/java/io/aiven/kafka/connect/transforms/Hash.java @@ -18,7 +18,6 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.Base64; import java.util.Map; import java.util.Optional; @@ -32,6 +31,8 @@ import org.apache.kafka.connect.errors.DataException; import org.apache.kafka.connect.transforms.Transformation; +import io.aiven.kafka.connect.transforms.utils.Hex; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -178,8 +179,9 @@ private Optional getNewValueWithoutFieldName(final String recordStr, } private String hashString(final String string) { + // We don't call reset() here because digest() does resetting afterwards. final byte[] digest = messageDigest.digest(string.getBytes()); - return Base64.getEncoder().encodeToString(digest); + return Hex.encode(digest); } public static class Key> extends Hash { diff --git a/src/main/java/io/aiven/kafka/connect/transforms/utils/Hex.java b/src/main/java/io/aiven/kafka/connect/transforms/utils/Hex.java new file mode 100644 index 0000000..cff7dfc --- /dev/null +++ b/src/main/java/io/aiven/kafka/connect/transforms/utils/Hex.java @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Aiven Oy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.aiven.kafka.connect.transforms.utils; + +public class Hex { + private static final char[] HEX_ARRAY = "0123456789abcdef".toCharArray(); + + /** + * Encodes a byte array as a hexadecimal string. + * @implNote https://stackoverflow.com/a/9855338/1781549 + */ + public static String encode(final byte[] bytes) { + final char[] hexChars = new char[bytes.length * 2]; + for (int j = 0; j < bytes.length; j++) { + final int v = bytes[j] & 0xFF; + hexChars[j * 2] = HEX_ARRAY[v >>> 4]; // hi nibble + hexChars[j * 2 + 1] = HEX_ARRAY[v & 0x0F]; // lo nibble + } + return new String(hexChars); + } +} diff --git a/src/test/java/io/aiven/kafka/connect/transforms/HashTest.java b/src/test/java/io/aiven/kafka/connect/transforms/HashTest.java index 597112d..ffbf24d 100644 --- a/src/test/java/io/aiven/kafka/connect/transforms/HashTest.java +++ b/src/test/java/io/aiven/kafka/connect/transforms/HashTest.java @@ -16,9 +16,6 @@ package io.aiven.kafka.connect.transforms; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Base64; import java.util.HashMap; import java.util.Map; @@ -40,7 +37,23 @@ abstract class HashTest { private static final String FIELD = "email"; private static final String EMPTY_FIELD_VALUE = ""; - private static final String NON_EMPTY_FIELD_VALUE = "jerry@all_your_bases.com"; + // echo -n "" | md5sum -t + private static final String EMPTY_FIELD_VALUE_MD5 = "d41d8cd98f00b204e9800998ecf8427e"; + // echo -n "" | sha1sum -t + private static final String EMPTY_FIELD_VALUE_SHA1 = "da39a3ee5e6b4b0d3255bfef95601890afd80709"; + // echo -n "" | sha256sum -t + private static final String EMPTY_FIELD_VALUE_SHA256 = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + private static final String NON_EMPTY_FIELD_VALUE = "jerry@big-corp.com"; + // echo -n "jerry@big-corp.com" | md5sum -t + private static final String NON_EMPTY_FIELD_VALUE_MD5 = "10e5756d5d4c9c1cadd5e1b952071378"; + // echo -n "jerry@big-corp.com" | sha1sum -t + private static final String NON_EMPTY_FIELD_VALUE_SHA1 = "dd9ab6e93603bf618db0894a82da64f1623a94b6"; + // echo -n "jerry@big-corp.com" | sha256sum -t + private static final String NON_EMPTY_FIELD_VALUE_SHA256 = + "20e85b05e7349963fc64746fbc7f3f4fdf31507921360847ebef333b229cf2d6"; + private static final String DEFAULT_HASH_FUNCTION = HashConfig.HashFunction.SHA256.toString(); private static final String UNAFFECTED_FIELD = "name"; private static final String UNAFFECTED_FIELD_VALUE = "jerry"; @@ -232,6 +245,20 @@ void fieldName_EmptyStringValue(final String hashFunction) { assertEquals(setNewValue(originalRecord, newValue), result); } + @ParameterizedTest + @ValueSource(strings = {"md5", "sha1", "sha256"}) + void sameValueSameHash(final String hashFunction) { + final Schema schema = SchemaBuilder.STRING_SCHEMA; + final Hash transform = transformation(null, false, hashFunction); + + for (int i = 0; i < 10; i++) { + final SinkRecord originalRecord = record(schema, NON_EMPTY_FIELD_VALUE); + final SinkRecord result = transform.apply(originalRecord); + final String newValue = hash(hashFunction, NON_EMPTY_FIELD_VALUE); + assertEquals(setNewValue(originalRecord, newValue), result); + } + } + private Hash transformation( final String fieldName, final boolean skipMissingOrNull, @@ -277,24 +304,39 @@ private SinkRecord setNewValue(final SinkRecord record, final Object newValue) { } private String hash(final String function, final String value) { - try { - final MessageDigest md; - switch (function) { - case "md5": - md = MessageDigest.getInstance("MD5"); - break; - case "sha1": - md = MessageDigest.getInstance("SHA1"); - break; - case "sha256": - md = MessageDigest.getInstance("SHA-256"); - break; - default: - throw new IllegalArgumentException(function); - } - return Base64.getEncoder().encodeToString(md.digest(value.getBytes())); - } catch (final NoSuchAlgorithmException e) { - throw new RuntimeException(e); + switch (function) { + case "md5": + switch (value) { + case EMPTY_FIELD_VALUE: + return EMPTY_FIELD_VALUE_MD5; + case NON_EMPTY_FIELD_VALUE: + return NON_EMPTY_FIELD_VALUE_MD5; + default: + throw new IllegalArgumentException(value); + } + + case "sha1": + switch (value) { + case EMPTY_FIELD_VALUE: + return EMPTY_FIELD_VALUE_SHA1; + case NON_EMPTY_FIELD_VALUE: + return NON_EMPTY_FIELD_VALUE_SHA1; + default: + throw new IllegalArgumentException(value); + } + + case "sha256": + switch (value) { + case EMPTY_FIELD_VALUE: + return EMPTY_FIELD_VALUE_SHA256; + case NON_EMPTY_FIELD_VALUE: + return NON_EMPTY_FIELD_VALUE_SHA256; + default: + throw new IllegalArgumentException(value); + } + + default: + throw new IllegalArgumentException(function); } } } diff --git a/src/test/java/io/aiven/kafka/connect/transforms/utils/HexTest.java b/src/test/java/io/aiven/kafka/connect/transforms/utils/HexTest.java new file mode 100644 index 0000000..fb1a1f1 --- /dev/null +++ b/src/test/java/io/aiven/kafka/connect/transforms/utils/HexTest.java @@ -0,0 +1,68 @@ +/* + * Copyright 2020 Aiven Oy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.aiven.kafka.connect.transforms.utils; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class HexTest { + @Test + void testEncodeEmpty() { + final byte[] bytes = new byte[0]; + assertEquals("", Hex.encode(bytes)); + } + + @Test + void testEncodeSingleByte() { + final byte[] bytes = new byte[1]; + for (int i = 0; i < 256; i++) { + final byte b = (byte) i; + bytes[0] = b; + assertEquals(String.format("%02x", b), Hex.encode(bytes)); + } + } + + @Test + void testEncodeFromStrings() throws IOException, URISyntaxException { + final URL resource = getClass().getClassLoader().getResource("blns.txt"); + final List strings = Files.readAllLines(Paths.get(resource.toURI())); + for (final String s : strings) { + // Use the string as a byte array and hex-encode it. + final byte[] bytes = s.getBytes(Charset.defaultCharset()); + final String encoded = Hex.encode(bytes); + assertEquals(bytes.length * 2, encoded.length()); + + // Decode the string back and compare to the original. + final char[] encodedChars = encoded.toCharArray(); + final byte[] decodedBytes = new byte[bytes.length]; + for (int i = 0; i < encoded.length(); i += 2) { + final String s1 = new String(encodedChars, i, 2); + decodedBytes[i / 2] = (byte) Integer.parseInt(s1, 16); + } + assertEquals(new String(decodedBytes, Charset.defaultCharset()), s); + } + } +} diff --git a/src/test/resources/blns.txt b/src/test/resources/blns.txt new file mode 100644 index 0000000..e8d7d2b --- /dev/null +++ b/src/test/resources/blns.txt @@ -0,0 +1,744 @@ +# https://github.com/minimaxir/big-list-of-naughty-strings + +# Reserved Strings +# +# Strings which may be used elsewhere in code + +undefined +undef +null +NULL +(null) +nil +NIL +true +false +True +False +TRUE +FALSE +None +hasOwnProperty +then +constructor +\ +\\ + +# Numeric Strings +# +# Strings which can be interpreted as numeric + +0 +1 +1.00 +$1.00 +1/2 +1E2 +1E02 +1E+02 +-1 +-1.00 +-$1.00 +-1/2 +-1E2 +-1E02 +-1E+02 +1/0 +0/0 +-2147483648/-1 +-9223372036854775808/-1 +-0 +-0.0 ++0 ++0.0 +0.00 +0..0 +. +0.0.0 +0,00 +0,,0 +, +0,0,0 +0.0/0 +1.0/0.0 +0.0/0.0 +1,0/0,0 +0,0/0,0 +--1 +- +-. +-, +999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999 +NaN +Infinity +-Infinity +INF +1#INF +-1#IND +1#QNAN +1#SNAN +1#IND +0x0 +0xffffffff +0xffffffffffffffff +0xabad1dea +123456789012345678901234567890123456789 +1,000.00 +1 000.00 +1'000.00 +1,000,000.00 +1 000 000.00 +1'000'000.00 +1.000,00 +1 000,00 +1'000,00 +1.000.000,00 +1 000 000,00 +1'000'000,00 +01000 +08 +09 +2.2250738585072011e-308 + +# Special Characters +# +# ASCII punctuation. All of these characters may need to be escaped in some +# contexts. Divided into three groups based on (US-layout) keyboard position. + +,./;'[]\-= +<>?:"{}|_+ +!@#$%^&*()`~ + +# Non-whitespace C0 controls: U+0001 through U+0008, U+000E through U+001F, +# and U+007F (DEL) +# Often forbidden to appear in various text-based file formats (e.g. XML), +# or reused for internal delimiters on the theory that they should never +# appear in input. +# The next line may appear to be blank or mojibake in some viewers. + + +# Non-whitespace C1 controls: U+0080 through U+0084 and U+0086 through U+009F. +# Commonly misinterpreted as additional graphic characters. +# The next line may appear to be blank, mojibake, or dingbats in some viewers. +€‚ƒ„†‡ˆ‰Š‹ŒŽ‘’“”•–—˜™š›œžŸ + +# Whitespace: all of the characters with category Zs, Zl, or Zp (in Unicode +# version 8.0.0), plus U+0009 (HT), U+000B (VT), U+000C (FF), U+0085 (NEL), +# and U+200B (ZERO WIDTH SPACE), which are in the C categories but are often +# treated as whitespace in some contexts. +# This file unfortunately cannot express strings containing +# U+0000, U+000A, or U+000D (NUL, LF, CR). +# The next line may appear to be blank or mojibake in some viewers. +# The next line may be flagged for "trailing whitespace" in some viewers. + …             ​

  　 + +# Unicode additional control characters: all of the characters with +# general category Cf (in Unicode 8.0.0). +# The next line may appear to be blank or mojibake in some viewers. +­؀؁؂؃؄؅؜۝܏᠎​‌‍‎‏‪‫‬‭‮⁠⁡⁢⁣⁤⁦⁧⁨⁩𑂽𛲠𛲡𛲢𛲣𝅳𝅴𝅵𝅶𝅷𝅸𝅹𝅺󠀁󠀠󠀡󠀢󠀣󠀤󠀥󠀦󠀧󠀨󠀩󠀪󠀫󠀬󠀭󠀮󠀯󠀰󠀱󠀲󠀳󠀴󠀵󠀶󠀷󠀸󠀹󠀺󠀻󠀼󠀽󠀾󠀿󠁀󠁁󠁂󠁃󠁄󠁅󠁆󠁇󠁈󠁉󠁊󠁋󠁌󠁍󠁎󠁏󠁐󠁑󠁒󠁓󠁔󠁕󠁖󠁗󠁘󠁙󠁚󠁛󠁜󠁝󠁞󠁟󠁠󠁡󠁢󠁣󠁤󠁥󠁦󠁧󠁨󠁩󠁪󠁫󠁬󠁭󠁮󠁯󠁰󠁱󠁲󠁳󠁴󠁵󠁶󠁷󠁸󠁹󠁺󠁻󠁼󠁽󠁾󠁿 + +# "Byte order marks", U+FEFF and U+FFFE, each on its own line. +# The next two lines may appear to be blank or mojibake in some viewers. + +￾ + +# Unicode Symbols +# +# Strings which contain common unicode symbols (e.g. smart quotes) + +Ω≈ç√∫˜µ≤≥÷ +åß∂ƒ©˙∆˚¬…æ +œ∑´®†¥¨ˆøπ“‘ +¡™£¢∞§¶•ªº–≠ +¸˛Ç◊ı˜Â¯˘¿ +ÅÍÎÏ˝ÓÔÒÚÆ☃ +Œ„´‰ˇÁ¨ˆØ∏”’ +`⁄€‹›fifl‡°·‚—± +⅛⅜⅝⅞ +ЁЂЃЄЅІЇЈЉЊЋЌЍЎЏАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя +٠١٢٣٤٥٦٧٨٩ + +# Unicode Subscript/Superscript/Accents +# +# Strings which contain unicode subscripts/superscripts; can cause rendering issues + +⁰⁴⁵ +₀₁₂ +⁰⁴⁵₀₁₂ +ด้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็ ด้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็ ด้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็็้้้้้้้้็็็็็้้้้้็็็็ + +# Quotation Marks +# +# Strings which contain misplaced quotation marks; can cause encoding errors + +' +" +'' +"" +'"' +"''''"'" +"'"'"''''" + + + + + +# Two-Byte Characters +# +# Strings which contain two-byte characters: can cause rendering issues or character-length issues + +田中さんにあげて下さい +パーティーへ行かないか +和製漢語 +部落格 +사회과학원 어학연구소 +찦차를 타고 온 펲시맨과 쑛다리 똠방각하 +社會科學院語學研究所 +울란바토르 +𠜎𠜱𠝹𠱓𠱸𠲖𠳏 + +# Strings which contain two-byte letters: can cause issues with naïve UTF-16 capitalizers which think that 16 bits == 1 character + +𐐜 𐐔𐐇𐐝𐐀𐐡𐐇𐐓 𐐙𐐊𐐡𐐝𐐓/𐐝𐐇𐐗𐐊𐐤𐐔 𐐒𐐋𐐗 𐐒𐐌 𐐜 𐐡𐐀𐐖𐐇𐐤𐐓𐐝 𐐱𐑂 𐑄 𐐔𐐇𐐝𐐀𐐡𐐇𐐓 𐐏𐐆𐐅𐐤𐐆𐐚𐐊𐐡𐐝𐐆𐐓𐐆 + +# Special Unicode Characters Union +# +# A super string recommended by VMware Inc. Globalization Team: can effectively cause rendering issues or character-length issues to validate product globalization readiness. +# +# 表 CJK_UNIFIED_IDEOGRAPHS (U+8868) +# ポ KATAKANA LETTER PO (U+30DD) +# あ HIRAGANA LETTER A (U+3042) +# A LATIN CAPITAL LETTER A (U+0041) +# 鷗 CJK_UNIFIED_IDEOGRAPHS (U+9DD7) +# Œ LATIN SMALL LIGATURE OE (U+0153) +# é LATIN SMALL LETTER E WITH ACUTE (U+00E9) +# Ｂ FULLWIDTH LATIN CAPITAL LETTER B (U+FF22) +# 逍 CJK_UNIFIED_IDEOGRAPHS (U+900D) +# Ü LATIN SMALL LETTER U WITH DIAERESIS (U+00FC) +# ß LATIN SMALL LETTER SHARP S (U+00DF) +# ª FEMININE ORDINAL INDICATOR (U+00AA) +# ą LATIN SMALL LETTER A WITH OGONEK (U+0105) +# ñ LATIN SMALL LETTER N WITH TILDE (U+00F1) +# 丂 CJK_UNIFIED_IDEOGRAPHS (U+4E02) +# 㐀 CJK Ideograph Extension A, First (U+3400) +# 𠀀 CJK Ideograph Extension B, First (U+20000) + +表ポあA鷗ŒéＢ逍Üߪąñ丂㐀𠀀 + +# Changing length when lowercased +# +# Characters which increase in length (2 to 3 bytes) when lowercased +# Credit: https://twitter.com/jifa/status/625776454479970304 + +Ⱥ +Ⱦ + +# Japanese Emoticons +# +# Strings which consists of Japanese-style emoticons which are popular on the web + +ヽ༼ຈل͜ຈ༽ﾉ ヽ༼ຈل͜ຈ༽ﾉ +(｡◕ ∀ ◕｡) +｀ｨ(´∀｀∩ +__ﾛ(,_,*) +・(￣∀￣)・:*: +ﾟ･✿ヾ╲(｡◕‿◕｡)╱✿･ﾟ +,。・:*:・゜’( ☻ ω ☻ )。・:*:・゜’ +(╯°□°）╯︵ ┻━┻) +(ﾉಥ益ಥ）ﾉ ┻━┻ +┬─┬ノ( º _ ºノ) +( ͡° ͜ʖ ͡°) +¯\_(ツ)_/¯ + +# Emoji +# +# Strings which contain Emoji; should be the same behavior as two-byte characters, but not always + +😍 +👩🏽 +👨‍🦰 👨🏿‍🦰 👨‍🦱 👨🏿‍🦱 🦹🏿‍♂️ +👾 🙇 💁 🙅 🙆 🙋 🙎 🙍 +🐵 🙈 🙉 🙊 +❤️ 💔 💌 💕 💞 💓 💗 💖 💘 💝 💟 💜 💛 💚 💙 +✋🏿 💪🏿 👐🏿 🙌🏿 👏🏿 🙏🏿 +👨‍👩‍👦 👨‍👩‍👧‍👦 👨‍👨‍👦 👩‍👩‍👧 👨‍👦 👨‍👧‍👦 👩‍👦 👩‍👧‍👦 +🚾 🆒 🆓 🆕 🆖 🆗 🆙 🏧 +0️⃣ 1️⃣ 2️⃣ 3️⃣ 4️⃣ 5️⃣ 6️⃣ 7️⃣ 8️⃣ 9️⃣ 🔟 + +# Regional Indicator Symbols +# +# Regional Indicator Symbols can be displayed differently across +# fonts, and have a number of special behaviors + +🇺🇸🇷🇺🇸 🇦🇫🇦🇲🇸 +🇺🇸🇷🇺🇸🇦🇫🇦🇲 +🇺🇸🇷🇺🇸🇦 + +# Unicode Numbers +# +# Strings which contain unicode numbers; if the code is localized, it should see the input as numeric + +１２３ +١٢٣ + +# Right-To-Left Strings +# +# Strings which contain text that should be rendered RTL if possible (e.g. Arabic, Hebrew) + +ثم نفس سقطت وبالتحديد،, جزيرتي باستخدام أن دنو. إذ هنا؟ الستار وتنصيب كان. أهّل ايطاليا، بريطانيا-فرنسا قد أخذ. سليمان، إتفاقية بين ما, يذكر الحدود أي بعد, معاملة بولندا، الإطلاق عل إيو. +בְּרֵאשִׁית, בָּרָא אֱלֹהִים, אֵת הַשָּׁמַיִם, וְאֵת הָאָרֶץ +הָיְתָהtestالصفحات التّحول +﷽ +ﷺ +مُنَاقَشَةُ سُبُلِ اِسْتِخْدَامِ اللُّغَةِ فِي النُّظُمِ الْقَائِمَةِ وَفِيم يَخُصَّ التَّطْبِيقَاتُ الْحاسُوبِيَّةُ، +الكل في المجمو عة (5) + +# Ogham Text +# +# The only unicode alphabet to use a space which isn't empty but should still act like a space. + +᚛ᚄᚓᚐᚋᚒᚄ ᚑᚄᚂᚑᚏᚅ᚜ +᚛                 ᚜ + +# Trick Unicode +# +# Strings which contain unicode with unusual properties (e.g. Right-to-left override) (c.f. http://www.unicode.org/charts/PDF/U2000.pdf) + +‪‪test‪ +‫test‫ +
test
 +test⁠test‫ +⁦test⁧ + +# Zalgo Text +# +# Strings which contain "corrupted" text. The corruption will not appear in non-HTML text, however. (via http://www.eeemo.net) + +Ṱ̺̺̕o͞ ̷i̲̬͇̪͙n̝̗͕v̟̜̘̦͟o̶̙̰̠kè͚̮̺̪̹̱̤ ̖t̝͕̳̣̻̪͞h̼͓̲̦̳̘̲e͇̣̰̦̬͎ ̢̼̻̱̘h͚͎͙̜̣̲ͅi̦̲̣̰̤v̻͍e̺̭̳̪̰-m̢iͅn̖̺̞̲̯̰d̵̼̟͙̩̼̘̳ ̞̥̱̳̭r̛̗̘e͙p͠r̼̞̻̭̗e̺̠̣͟s̘͇̳͍̝͉e͉̥̯̞̲͚̬͜ǹ̬͎͎̟̖͇̤t͍̬̤͓̼̭͘ͅi̪̱n͠g̴͉ ͏͉ͅc̬̟h͡a̫̻̯͘o̫̟̖͍̙̝͉s̗̦̲.̨̹͈̣ +̡͓̞ͅI̗̘̦͝n͇͇͙v̮̫ok̲̫̙͈i̖͙̭̹̠̞n̡̻̮̣̺g̲͈͙̭͙̬͎ ̰t͔̦h̞̲e̢̤ ͍̬̲͖f̴̘͕̣è͖ẹ̥̩l͖͔͚i͓͚̦͠n͖͍̗͓̳̮g͍ ̨o͚̪͡f̘̣̬ ̖̘͖̟͙̮c҉͔̫͖͓͇͖ͅh̵̤̣͚͔á̗̼͕ͅo̼̣̥s̱͈̺̖̦̻͢.̛̖̞̠̫̰ +̗̺͖̹̯͓Ṯ̤͍̥͇͈h̲́e͏͓̼̗̙̼̣͔ ͇̜̱̠͓͍ͅN͕͠e̗̱z̘̝̜̺͙p̤̺̹͍̯͚e̠̻̠͜r̨̤͍̺̖͔̖̖d̠̟̭̬̝͟i̦͖̩͓͔̤a̠̗̬͉̙n͚͜ ̻̞̰͚ͅh̵͉i̳̞v̢͇ḙ͎͟-҉̭̩̼͔m̤̭̫i͕͇̝̦n̗͙ḍ̟ ̯̲͕͞ǫ̟̯̰̲͙̻̝f ̪̰̰̗̖̭̘͘c̦͍̲̞͍̩̙ḥ͚a̮͎̟̙͜ơ̩̹͎s̤.̝̝ ҉Z̡̖̜͖̰̣͉̜a͖̰͙̬͡l̲̫̳͍̩g̡̟̼̱͚̞̬ͅo̗͜.̟ +̦H̬̤̗̤͝e͜ ̜̥̝̻͍̟́w̕h̖̯͓o̝͙̖͎̱̮ ҉̺̙̞̟͈W̷̼̭a̺̪͍į͈͕̭͙̯̜t̶̼̮s̘͙͖̕ ̠̫̠B̻͍͙͉̳ͅe̵h̵̬͇̫͙i̹͓̳̳̮͎̫̕n͟d̴̪̜̖ ̰͉̩͇͙̲͞ͅT͖̼͓̪͢h͏͓̮̻e̬̝̟ͅ ̤̹̝W͙̞̝͔͇͝ͅa͏͓͔̹̼̣l̴͔̰̤̟͔ḽ̫.͕ +Z̮̞̠͙͔ͅḀ̗̞͈̻̗Ḷ͙͎̯̹̞͓G̻O̭̗̮ + +# Unicode Upsidedown +# +# Strings which contain unicode with an "upsidedown" effect (via http://www.upsidedowntext.com) + +˙ɐnbᴉlɐ ɐuƃɐɯ ǝɹolop ʇǝ ǝɹoqɐl ʇn ʇunpᴉpᴉɔuᴉ ɹodɯǝʇ poɯsnᴉǝ op pǝs 'ʇᴉlǝ ƃuᴉɔsᴉdᴉpɐ ɹnʇǝʇɔǝsuoɔ 'ʇǝɯɐ ʇᴉs ɹolop ɯnsdᴉ ɯǝɹo˥ +00˙Ɩ$- + +# Unicode font +# +# Strings which contain bold/italic/etc. versions of normal characters + +Ｔｈｅ ｑｕｉｃｋ ｂｒｏｗｎ ｆｏｘ ｊｕｍｐｓ ｏｖｅｒ ｔｈｅ ｌａｚｙ ｄｏｇ +𝐓𝐡𝐞 𝐪𝐮𝐢𝐜𝐤 𝐛𝐫𝐨𝐰𝐧 𝐟𝐨𝐱 𝐣𝐮𝐦𝐩𝐬 𝐨𝐯𝐞𝐫 𝐭𝐡𝐞 𝐥𝐚𝐳𝐲 𝐝𝐨𝐠 +𝕿𝖍𝖊 𝖖𝖚𝖎𝖈𝖐 𝖇𝖗𝖔𝖜𝖓 𝖋𝖔𝖝 𝖏𝖚𝖒𝖕𝖘 𝖔𝖛𝖊𝖗 𝖙𝖍𝖊 𝖑𝖆𝖟𝖞 𝖉𝖔𝖌 +𝑻𝒉𝒆 𝒒𝒖𝒊𝒄𝒌 𝒃𝒓𝒐𝒘𝒏 𝒇𝒐𝒙 𝒋𝒖𝒎𝒑𝒔 𝒐𝒗𝒆𝒓 𝒕𝒉𝒆 𝒍𝒂𝒛𝒚 𝒅𝒐𝒈 +𝓣𝓱𝓮 𝓺𝓾𝓲𝓬𝓴 𝓫𝓻𝓸𝔀𝓷 𝓯𝓸𝔁 𝓳𝓾𝓶𝓹𝓼 𝓸𝓿𝓮𝓻 𝓽𝓱𝓮 𝓵𝓪𝔃𝔂 𝓭𝓸𝓰 +𝕋𝕙𝕖 𝕢𝕦𝕚𝕔𝕜 𝕓𝕣𝕠𝕨𝕟 𝕗𝕠𝕩 𝕛𝕦𝕞𝕡𝕤 𝕠𝕧𝕖𝕣 𝕥𝕙𝕖 𝕝𝕒𝕫𝕪 𝕕𝕠𝕘 +𝚃𝚑𝚎 𝚚𝚞𝚒𝚌𝚔 𝚋𝚛𝚘𝚠𝚗 𝚏𝚘𝚡 𝚓𝚞𝚖𝚙𝚜 𝚘𝚟𝚎𝚛 𝚝𝚑𝚎 𝚕𝚊𝚣𝚢 𝚍𝚘𝚐 +⒯⒣⒠ ⒬⒰⒤⒞⒦ ⒝⒭⒪⒲⒩ ⒡⒪⒳ ⒥⒰⒨⒫⒮ ⒪⒱⒠⒭ ⒯⒣⒠ ⒧⒜⒵⒴ ⒟⒪⒢ + +# Script Injection +# +# Strings which attempt to invoke a benign script injection; shows vulnerability to XSS + + +<script>alert('123');</script> + + +"> +'> +> + +< / script >< script >alert(123)< / script > + onfocus=JaVaSCript:alert(123) autofocus +" onfocus=JaVaSCript:alert(123) autofocus +' onfocus=JaVaSCript:alert(123) autofocus +＜script＞alert(123)＜/script＞ +ript>alert(123)ript> +--> +";alert(123);t=" +';alert(123);t=' +JavaSCript:alert(123) +;alert(123); +src=JaVaSCript:prompt(132) +">javascript:alert(1); +javascript:alert(1); +javascript:alert(1); +javascript:alert(1); +javascript:alert(1); +javascript:alert(1); +javascript:alert(1); +'`"><\x3Cscript>javascript:alert(1) +'`"><\x00script>javascript:alert(1) +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +ABC
DEF +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +test +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +`"'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> +"`'> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +XXX + + + +<a href=http://foo.bar/#x=`y></a><img alt="`><img src=x:x onerror=javascript:alert(1)></a>"> +<!--[if]><script>javascript:alert(1)</script --> +<!--[if<img src=x onerror=javascript:alert(1)//]> --> +<script src="/\%(jscript)s"></script> +<script src="\\%(jscript)s"></script> +<IMG """><SCRIPT>alert("XSS")</SCRIPT>"> +<IMG SRC=javascript:alert(String.fromCharCode(88,83,83))> +<IMG SRC=# onmouseover="alert('xxs')"> +<IMG SRC= onmouseover="alert('xxs')"> +<IMG onmouseover="alert('xxs')"> +<IMG SRC=javascript:alert('XSS')> +<IMG SRC=javascript:alert('XSS')> +<IMG SRC=javascript:alert('XSS')> +<IMG SRC="jav ascript:alert('XSS');"> +<IMG SRC="jav ascript:alert('XSS');"> +<IMG SRC="jav ascript:alert('XSS');"> +<IMG SRC="jav ascript:alert('XSS');"> +perl -e 'print "<IMG SRC=java\0script:alert(\"XSS\")>";' > out +<IMG SRC="  javascript:alert('XSS');"> +<SCRIPT/XSS SRC="http://ha.ckers.org/xss.js"></SCRIPT> +<BODY onload!#$%&()*~+-_.,:;?@[/|\]^`=alert("XSS")> +<SCRIPT/SRC="http://ha.ckers.org/xss.js"></SCRIPT> +<<SCRIPT>alert("XSS");//<</SCRIPT> +<SCRIPT SRC=http://ha.ckers.org/xss.js?< B > +<SCRIPT SRC=//ha.ckers.org/.j> +<IMG SRC="javascript:alert('XSS')" +<iframe src=http://ha.ckers.org/scriptlet.html < +\";alert('XSS');// +<u oncopy=alert()> Copy me</u> +<i onwheel=alert(1)> Scroll over me </i> +<plaintext> +http://a/%%30%30 +</textarea><script>alert(123)</script> + +# SQL Injection +# +# Strings which can cause a SQL injection if inputs are not sanitized + +1;DROP TABLE users +1'; DROP TABLE users-- 1 +' OR 1=1 -- 1 +' OR '1'='1 +'; EXEC sp_MSForEachTable 'DROP TABLE ?'; -- + +% +_ + +# Server Code Injection +# +# Strings which can cause user to run code on server as a privileged user (c.f. https://news.ycombinator.com/item?id=7665153) + +- +-- +--version +--help +$USER +/dev/null; touch /tmp/blns.fail ; echo +`touch /tmp/blns.fail` +$(touch /tmp/blns.fail) +@{[system "touch /tmp/blns.fail"]} + +# Command Injection (Ruby) +# +# Strings which can call system commands within Ruby/Rails applications + +eval("puts 'hello world'") +System("ls -al /") +`ls -al /` +Kernel.exec("ls -al /") +Kernel.exit(1) +%x('ls -al /') + +# XXE Injection (XML) +# +# String which can reveal system files when parsed by a badly configured XML parser + +<?xml version="1.0" encoding="ISO-8859-1"?><!DOCTYPE foo [ <!ELEMENT foo ANY ><!ENTITY xxe SYSTEM "file:///etc/passwd" >]><foo>&xxe;</foo> + +# Unwanted Interpolation +# +# Strings which can be accidentally expanded into different strings if evaluated in the wrong context, e.g. used as a printf format string or via Perl or shell eval. Might expose sensitive data from the program doing the interpolation, or might just represent the wrong string. + +$HOME +$ENV{'HOME'} +%d +%s%s%s%s%s +{0} +%*.*s +%@ +%n +File:/// + +# File Inclusion +# +# Strings which can cause user to pull in files that should not be a part of a web server + +../../../../../../../../../../../etc/passwd%00 +../../../../../../../../../../../etc/hosts + +# Known CVEs and Vulnerabilities +# +# Strings that test for known vulnerabilities + +() { 0; }; touch /tmp/blns.shellshock1.fail; +() { _; } >_[$($())] { touch /tmp/blns.shellshock2.fail; } +<<< %s(un='%s') = %u ++++ATH0 + +# MSDOS/Windows Special Filenames +# +# Strings which are reserved characters in MSDOS/Windows + +CON +PRN +AUX +CLOCK$ +NUL +A: +ZZ: +COM1 +LPT1 +LPT2 +LPT3 +COM2 +COM3 +COM4 + +# IRC specific strings +# +# Strings that may occur on IRC clients that make security products freak out + +DCC SEND STARTKEYLOGGER 0 0 0 + +# Scunthorpe Problem +# +# Innocuous strings which may be blocked by profanity filters (https://en.wikipedia.org/wiki/Scunthorpe_problem) + +Scunthorpe General Hospital +Penistone Community Church +Lightwater Country Park +Jimmy Clitheroe +Horniman Museum +shitake mushrooms +RomansInSussex.co.uk +http://www.cum.qc.ca/ +Craig Cockburn, Software Specialist +Linda Callahan +Dr. Herman I. Libshitz +magna cum laude +Super Bowl XXX +medieval erection of parapets +evaluate +mocha +expression +Arsenal canal +classic +Tyson Gay +Dick Van Dyke +basement + +# Human injection +# +# Strings which may cause human to reinterpret worldview + +If you're reading this, you've been in a coma for almost 20 years now. We're trying a new technique. We don't know where this message will end up in your dream, but we hope it works. Please wake up, we miss you. + +# Terminal escape codes +# +# Strings which punish the fools who use cat/type on this file + +Roses are red, violets are blue. Hope you enjoy terminal hue +But now...for my greatest trick... +The quick brown fox... [Beeeep] + +# iOS Vulnerabilities +# +# Strings which crashed iMessage in various versions of iOS + +Powerلُلُصّبُلُلصّبُررً ॣ ॣh ॣ ॣ冗 +🏳0🌈️ +జ్ఞ‌ా + +# Persian special characters +# +# This is a four characters string which includes Persian special characters (گچپژ) + +گچپژ + +# jinja2 injection +# +# first one is supposed to raise "MemoryError" exception +# second, obviously, prints contents of /etc/passwd + +{% print 'x' * 64 * 1024**3 %} +{{ "".__class__.__mro__[2].__subclasses__()[40]("/etc/passwd").read() }}