Supports UTF-8 file with BOM.

app/src/main/java/net/gsantner/markor/activity/DocumentEditFragment.java

@@ -64,7 +64,6 @@
 
 import butterknife.BindView;
 import butterknife.OnTextChanged;
-import other.writeily.widget.WrMarkorWidgetProvider;
 
 @SuppressWarnings({"UnusedReturnValue"})
 @SuppressLint("NonConstantResourceId")

app/src/main/java/net/gsantner/markor/activity/openeditor/OpenEditorActivity.java

@@ -21,6 +21,8 @@
 import net.gsantner.opoc.util.PermissionChecker;
 
 import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
 
 public class OpenEditorActivity extends MarkorBaseActivity {
     protected void openEditorForFile(File file) {
@@ -40,7 +42,9 @@ protected void openActivityAndClose(final Intent openIntent, File file) {
                     file.getParentFile().mkdirs();
                 }
                 if (!file.exists() && !file.isDirectory()) {
-                    FileUtils.writeFile(file, "");
+                    final Map<String, Object> options = new HashMap<>(1);
+                    options.put(FileUtils.WITH_BOM, new AppSettings(getApplicationContext()).getNewFileDialogLastUsedUtf8Bom());
+                    FileUtils.writeFile(file, "", options);
                 }
                 openIntent.putExtra(Document.EXTRA_PATH, openIntent.hasExtra(Document.EXTRA_PATH) ? openIntent.getSerializableExtra(Document.EXTRA_PATH) : file);
                 new ActivityUtils(this).animateToActivity(openIntent, true, 1).freeContextRef();

app/src/main/java/net/gsantner/markor/format/todotxt/TodoTxtTextActions.java

@@ -164,7 +164,7 @@ public boolean onActionClick(final @StringRes int action) {
                         File doneFile = new File(_document.getFile().getParentFile(), callbackPayload);
                         String doneFileContents = "";
                         if (doneFile.exists() && doneFile.canRead()) {
-                            doneFileContents = FileUtils.readTextFileFast(doneFile).trim() + "\n";
+                            doneFileContents = FileUtils.readTextFileFast(doneFile).first.trim() + "\n";
                         }
                         doneFileContents += TodoTxtTask.tasksToString(move) + "\n";
 

app/src/main/java/net/gsantner/markor/model/Document.java

@@ -16,6 +16,7 @@
 import android.support.annotation.StringRes;
 import android.text.TextUtils;
 import android.util.Log;
+import android.util.Pair;
 import android.widget.Toast;
 
 import net.gsantner.markor.R;
@@ -35,7 +36,9 @@
 import java.nio.file.Files;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.security.SecureRandom;
+import java.util.HashMap;
 import java.util.Locale;
+import java.util.Map;
 
 import other.de.stanetz.jpencconverter.JavaPasswordbasedCryption;
 
@@ -52,6 +55,7 @@ public class Document implements Serializable {
     private String _title = "";
     private String _path = "";
     private long _modTime = 0;
+    private boolean _withBom = false;
     @StringRes
     private int _format = TextFormat.FORMAT_UNKNOWN;
 
@@ -160,7 +164,6 @@ public boolean isContentSame(final CharSequence s) {
     }
 
     public synchronized String loadContent(final Context context) {
-
         String content;
         final char[] pw;
         if (isEncrypted() && (pw = getPasswordWithWarning(context)) != null) {
@@ -180,7 +183,9 @@ public synchronized String loadContent(final Context context) {
                 content = "";
             }
         } else {
-            content = FileUtils.readTextFileFast(_file);
+            final Pair<String, Map<String, Object>> result = FileUtils.readTextFileFast(_file);
+            content = result.first;
+            _withBom = (boolean) result.second.get(FileUtils.WITH_BOM);
         }
 
         if (MainActivity.IS_DEBUG_ENABLED) {
@@ -257,14 +262,21 @@ public synchronized boolean saveContent(final Context context, final String cont
             if (shareUtil.isUnderStorageAccessFolder(_file)) {
                 shareUtil.writeFile(_file, false, (fileOpened, fos) -> {
                     try {
+                        if (_withBom) {
+                            fos.write(0xEF);
+                            fos.write(0xBB);
+                            fos.write(0xBF);
+                        }
                         fos.write(contentAsBytes);
                         fos.flush();
                     } catch (Exception ignored) {
                     }
                 });
                 success = true;
             } else {
-                success = FileUtils.writeFile(_file, contentAsBytes);
+                final Map<String, Object> options = new HashMap<>(1);
+                options.put(FileUtils.WITH_BOM, _withBom);
+                success = FileUtils.writeFile(_file, contentAsBytes, options);
             }
         } catch (JavaPasswordbasedCryption.EncryptionFailedException e) {
             Log.e(Document.class.getName(), "writeContent:  encrypt failed for File " + getPath() + ". " + e.getMessage(), e);

app/src/main/java/net/gsantner/markor/ui/NewFileDialog.java

@@ -90,6 +90,7 @@ private AlertDialog.Builder makeDialog(final File basedir, final boolean allowCr
         final EditText fileNameEdit = root.findViewById(R.id.new_file_dialog__name);
         final EditText fileExtEdit = root.findViewById(R.id.new_file_dialog__ext);
         final CheckBox encryptCheckbox = root.findViewById(R.id.new_file_dialog__encrypt);
+        final CheckBox utf8BomCheckbox = root.findViewById(R.id.new_file_dialog__utf8_bom);
         final Spinner typeSpinner = root.findViewById(R.id.new_file_dialog__type);
         final Spinner templateSpinner = root.findViewById(R.id.new_file_dialog__template);
         final String[] typeSpinnerToExtension = getResources().getStringArray(R.array.new_file_types__file_extension);
@@ -99,6 +100,7 @@ private AlertDialog.Builder makeDialog(final File basedir, final boolean allowCr
         } else {
             encryptCheckbox.setVisibility(View.GONE);
         }
+        utf8BomCheckbox.setChecked(appSettings.getNewFileDialogLastUsedUtf8Bom());
         fileExtEdit.setText(appSettings.getNewFileDialogLastUsedExtension());
         fileNameEdit.requestFocus();
         new Handler().postDelayed(new ContextUtils.DoTouchView(fileNameEdit), 200);
@@ -154,6 +156,10 @@ private AlertDialog.Builder makeDialog(final File basedir, final boolean allowCr
             appSettings.setNewFileDialogLastUsedEncryption(isChecked);
         });
 
+        utf8BomCheckbox.setOnCheckedChangeListener((buttonView, isChecked) -> {
+            appSettings.setNewFileDialogLastUsedUtf8Bom(isChecked);
+        });
+
         dialogBuilder.setView(root);
         fileNameEdit.requestFocus();
 
@@ -171,6 +177,11 @@ private AlertDialog.Builder makeDialog(final File basedir, final boolean allowCr
                     final byte[] templateContents = getTemplateContent(templateSpinner, basedir, f.getName(), encryptCheckbox.isChecked());
                     shareUtil.writeFile(f, false, (arg_ok, arg_fos) -> {
                         try {
+                            if (appSettings.getNewFileDialogLastUsedUtf8Bom()) {
+                                arg_fos.write(0xEF);
+                                arg_fos.write(0xBB);
+                                arg_fos.write(0xBF);
+                            }
                             if (templateContents != null && (!f.exists() || f.length() < ShareUtil.MIN_OVERWRITE_LENGTH)) {
                                 arg_fos.write(templateContents);
                             }

app/src/main/java/net/gsantner/markor/ui/SearchOrCustomTextDialogCreator.java

@@ -670,7 +670,7 @@ public static void showInsertSnippetDialog(final Activity activity, final Callba
         dopt.isSearchEnabled = true;
         dopt.titleText = R.string.insert_snippet;
         dopt.messageText = Html.fromHtml("<small><small>" + folder.getAbsolutePath() + "</small></small>");
-        dopt.callback = (key) -> callback.callback(FileUtils.readTextFileFast(texts.get(key)));
+        dopt.callback = (key) -> callback.callback(FileUtils.readTextFileFast(texts.get(key)).first);
         SearchOrCustomTextDialog.showMultiChoiceDialogWithSearchFilterUI(activity, dopt);
     }
 

app/src/main/java/net/gsantner/markor/ui/fsearch/FileSearchResultSelectorDialog.java

@@ -4,13 +4,11 @@
 import android.app.Activity;
 import android.content.Context;
 import android.database.DataSetObserver;
-import android.graphics.Typeface;
 import android.support.v4.content.ContextCompat;
 import android.support.v4.util.Pair;
 import android.support.v7.app.AlertDialog;
 import android.support.v7.widget.AppCompatEditText;
 import android.text.Editable;
-import android.text.Html;
 import android.text.TextWatcher;
 import android.view.LayoutInflater;
 import android.view.View;
@@ -112,7 +110,8 @@ public void beforeTextChanged(final CharSequence arg0, final int arg1, final int
                 final int groupPosition = ExpandableListView.getPackedPositionGroup(position);
                 // Start on end of first line
                 dialogCallback.callback(((GroupItemsInfo) expandableListView.getExpandableListAdapter().getGroup(groupPosition)).path, 0);
-            };
+            }
+            ;
             return true;
         });
 

app/src/main/java/net/gsantner/markor/ui/hleditor/HighlightingEditor.java

@@ -224,7 +224,7 @@ private void highlightWithoutChange() {
                 if (MainActivity.IS_DEBUG_ENABLED) {
                     AppSettings.appendDebugLog("Start highlighting");
                 }
-                withAccessibilityDisabled(() ->_hl.run(getText()));
+                withAccessibilityDisabled(() -> _hl.run(getText()));
             } catch (Exception e) {
                 // In no case ever let highlighting crash the editor
                 e.printStackTrace();

app/src/main/java/net/gsantner/markor/util/AppSettings.java

@@ -834,6 +834,14 @@ public void setNewFileDialogLastUsedEncryption(boolean b) {
         setBool(R.string.pref_key__new_file_dialog_lastused_encryption, b);
     }
 
+    public boolean getNewFileDialogLastUsedUtf8Bom() {
+        return getBool(R.string.pref_key__new_file_dialog_lastused_encryption, false);
+    }
+
+    public void setNewFileDialogLastUsedUtf8Bom(boolean b) {
+        setBool(R.string.pref_key__new_file_dialog_lastused_encryption, b);
+    }
+
     public String getNewFileDialogLastUsedExtension() {
         return getString(R.string.pref_key__new_file_dialog_lastused_extension, ".md");
     }

app/src/main/java/net/gsantner/opoc/preference/GsPreferenceFragmentCompat.java

@@ -51,7 +51,6 @@
 import android.graphics.Rect;
 import android.os.Build;
 import android.os.Bundle;
-import android.os.Parcelable;
 import android.support.annotation.ColorInt;
 import android.support.annotation.DrawableRes;
 import android.support.annotation.NonNull;

app/src/main/java/net/gsantner/opoc/ui/FilesystemViewerFragment.java

@@ -40,7 +40,6 @@
 
 import net.gsantner.markor.R;
 import net.gsantner.markor.format.TextFormat;
-import net.gsantner.markor.model.Document;
 import net.gsantner.markor.ui.FileInfoDialog;
 import net.gsantner.markor.ui.FilesystemViewerCreator;
 import net.gsantner.markor.ui.SearchOrCustomTextDialogCreator;
@@ -66,7 +65,6 @@
 import other.writeily.model.WrMarkorSingleton;
 import other.writeily.ui.WrConfirmDialog;
 import other.writeily.ui.WrRenameDialog;
-import other.writeily.widget.WrMarkorWidgetProvider;
 
 public class FilesystemViewerFragment extends GsFragmentBase
         implements FilesystemViewerData.SelectionListener {
@@ -507,7 +505,7 @@ public boolean onOptionsItemSelected(MenuItem item) {
                 if (_filesystemViewerAdapter.areItemsSelected()) {
                     File file = new ArrayList<>(_filesystemViewerAdapter.getCurrentSelection()).get(0);
                     if (TextFormat.isTextFile(file)) {
-                        _shareUtil.setClipboard(FileUtils.readTextFileFast(file));
+                        _shareUtil.setClipboard(FileUtils.readTextFileFast(file).first);
                         Toast.makeText(getContext(), R.string.clipboard, Toast.LENGTH_SHORT).show();
                         _filesystemViewerAdapter.unselectAll();
                     }

app/src/main/java/net/gsantner/opoc/util/BackupUtils.java

@@ -178,7 +178,7 @@ public static void makeBackup(final Context context, final List<String> prefName
     @SuppressLint("ApplySharedPref")
     public static void loadBackup(final Context context, final File backupFileContainingJson) {
         try {
-            final JSONObject json = new JSONObject(FileUtils.readTextFileFast(backupFileContainingJson));
+            final JSONObject json = new JSONObject(FileUtils.readTextFileFast(backupFileContainingJson).first);
             final List<SharedPreferences.Editor> editors = new ArrayList<>();
             for (Iterator<String> it = json.keys(); it.hasNext(); ) {
                 final String prefName = it.next();

app/src/main/java/net/gsantner/opoc/util/FileUtils.java

@@ -12,6 +12,7 @@
 
 
 import android.text.TextUtils;
+import android.util.Pair;
 
 import java.io.BufferedInputStream;
 import java.io.BufferedReader;
@@ -32,8 +33,10 @@
 import java.text.DecimalFormatSymbols;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
@@ -44,20 +47,41 @@ public class FileUtils {
     // Used on methods like copyFile(src, dst)
     private static final int BUFFER_SIZE = 4096;
 
-    public static String readTextFileFast(final File file) {
+    public final static String WITH_BOM = "withBom";
+
+    public static Pair<String, Map<String, Object>> readTextFileFast(final File file) {
+        Map<String, Object> info = new HashMap<>(1);
+
         try (final FileInputStream inputStream = new FileInputStream(file)) {
             final ByteArrayOutputStream result = new ByteArrayOutputStream();
+
+            final byte[] bomBuffer = new byte[3];
+            int bomReadLength = inputStream.read(bomBuffer);
+            boolean withBom = bomReadLength == 3 &&
+                    bomBuffer[0] == (byte) 0xEF &&
+                    bomBuffer[1] == (byte) 0xBB &&
+                    bomBuffer[2] == (byte) 0xBF;
+            info.put(WITH_BOM, withBom);
+
+            if (!withBom && bomReadLength > 0) {
+                result.write(bomBuffer, 0, bomReadLength);
+            }
+            if (bomReadLength < 3) {
+                return new Pair<>(result.toString("UTF-8"), info);
+            }
+
             final byte[] buffer = new byte[1024];
             for (int length; (length = inputStream.read(buffer)) != -1; ) {
                 result.write(buffer, 0, length);
             }
-            return result.toString("UTF-8");
+            return new Pair<>(result.toString("UTF-8"), info);
         } catch (FileNotFoundException e) {
             System.err.println("readTextFileFast: File " + file + " not found.");
         } catch (IOException e) {
             e.printStackTrace();
         }
-        return "";
+
+        return new Pair<>("", info);
     }
 
     public static byte[] readCloseStreamWithSize(final InputStream stream, int size) {
@@ -177,8 +201,15 @@ public static byte[] readCloseBinaryStream(final InputStream stream) {
         return baos.toByteArray();
     }
 
-    public static boolean writeFile(final File file, final byte[] data) {
+    public static boolean writeFile(final File file, final byte[] data, final Map<String, Object> options) {
+        boolean withBom = options != null && (Boolean) options.get(WITH_BOM);
+
         try (final FileOutputStream output = new FileOutputStream(file)) {
+            if (withBom) {
+                output.write(0xEF);
+                output.write(0xBB);
+                output.write(0xBF);
+            }
             output.write(data);
             output.flush();
             return true;
@@ -188,8 +219,8 @@ public static boolean writeFile(final File file, final byte[] data) {
         }
     }
 
-    public static boolean writeFile(final File file, final String data) {
-        return writeFile(file, data.getBytes());
+    public static boolean writeFile(final File file, final String data, final Map<String, Object> options) {
+        return writeFile(file, data.getBytes(), options);
     }
 
     public static boolean copyFile(final File src, final File dst) {

app/src/main/java/net/gsantner/opoc/util/ShareUtil.java

@@ -796,7 +796,7 @@ public Object extractResultFromActivityResult(final int requestCode, final int r
 
                                 // Create temporary file in cache directory
                                 picturePath = File.createTempFile("image", "tmp", _context.getCacheDir()).getAbsolutePath();
-                                FileUtils.writeFile(new File(picturePath), FileUtils.readCloseBinaryStream(input));
+                                FileUtils.writeFile(new File(picturePath), FileUtils.readCloseBinaryStream(input), null);
                             }
                         } catch (IOException ignored) {
                             // nothing we can do here, null value will be handled below

app/src/main/java/net/gsantner/opoc/util/StringUtils.java

@@ -461,21 +461,21 @@ public static int[] findDiff(final CharSequence dest, final CharSequence source)
         final int minLength = Math.min(dl, sl);
 
         int start = 0;
-        while(start < minLength && source.charAt(start) == dest.charAt(start)) start++;
+        while (start < minLength && source.charAt(start) == dest.charAt(start)) start++;
 
         // Handle several special cases
         if (sl == dl && start == sl) { // Case where 2 sequences are same
-            return new int[] { sl, sl, sl };
+            return new int[]{sl, sl, sl};
         } else if (sl < dl && start == sl) { // Pure crop
-            return new int[] { sl, dl, sl };
+            return new int[]{sl, dl, sl};
         } else if (dl < sl && start == dl) { // Pure append
-            return new int[] { dl, dl, sl };
+            return new int[]{dl, dl, sl};
         }
 
         int end = 0;
         final int maxEnd = minLength - start;
-        while(end < maxEnd && source.charAt(sl - end - 1) == dest.charAt(dl - end - 1)) end++;
+        while (end < maxEnd && source.charAt(sl - end - 1) == dest.charAt(dl - end - 1)) end++;
 
-        return new int[] { start, dl - end, sl - end };
+        return new int[]{start, dl - end, sl - end};
     }
 }

app/src/main/res/layout/new_file_dialog.xml

@@ -119,6 +119,13 @@
             android:layout_marginTop="8dp"
             android:text="@string/encrypt_file_content" />
 
+        <CheckBox
+            android:id="@+id/new_file_dialog__utf8_bom"
+            android:layout_width="match_parent"
+            android:layout_height="wrap_content"
+            android:layout_marginTop="8dp"
+            android:text="@string/utf8_with_bom" />
+
         <TextView
             style="@style/TextAppearance.AppCompat.Caption"
             android:layout_width="match_parent"

app/src/main/res/values/strings.xml

@@ -391,6 +391,7 @@ work. If not, see <https://creativecommons.org/publicdomain/zero/1.0/>.
     <string name="file_encryption_password">File encryption password</string>
     <string name="no_password_set_cannot_encrypt_decrypt">No password set, cannot encrypt or decrypt.\nGo to Settings -> General -> Password to set a password.</string>
     <string name="encrypt_file_content">Encrypt file content</string>
+    <string name="utf8_with_bom">UTF-8 with BOM</string>
     <string name="could_not_encrypt_file_content_the_file_was_not_saved">Could not encrypt file content. The file was not saved.</string>
     <string name="could_not_decrypt_file_content_wrong_password_or_is_the_file_maybe_not_encrypted">Could not decrypt file content. Did you use a wrong password? Is the file really encrypted with Markors AES encryption?</string>
     <string name="password_already_set_setting_a_new_password_will_overwrite">Password already set. Setting a new password will overwrite the existing password and you may not be able to access files anymore in case you forget the password.</string>