From 6e66acf1e766c608640ba7b470daef15d3db5d7b Mon Sep 17 00:00:00 2001 From: "Denis Kuzmin [ github.com/3F ]" Date: Fri, 7 Dec 2018 22:22:09 +0300 Subject: [PATCH] Fixed support of the special symbols in path: `' &!~`@#$^(+)_=%-;[.]{,}`. Issue #7 * gnt.bat now uses safe logic for work with special symbols from `-msbuild` key. * And main logic just ignores `;` only when `|` is found. * Also fixes .packer for new logic with "|" and just adds .compressor from DllExport project. --- .vssbe | 12 ++ README.md | 2 +- caller/gnt.bat | 7 +- embedded/.compressor | 344 +++++++++++++++++++++++++++++++++++++++++++ embedded/.packer | 59 ++++++-- embedded/exec.tpl | 77 ++++++++-- embedded/sha1.cmd | 2 +- logic.targets | 7 +- 8 files changed, 473 insertions(+), 37 deletions(-) create mode 100644 embedded/.compressor diff --git a/.vssbe b/.vssbe index 1eece85..972fb37 100644 --- a/.vssbe +++ b/.vssbe @@ -108,6 +108,17 @@ "", "#[IO writeLine(STDOUT):#[File sout(\"netmsb.bat\", \"embedded/.packer /p:core=\\\"../$(dobj)gnt.min.core\\\" /p:output=\\\"../$(dobj)gnt.bat\\\" /nologo /v:m /m:4\", 400)]]", "", + "", + "#[\" ", + " .compressor for executable version (batch)", + "\"]", + "#[IO writeLine(STDOUT):#[File sout(\"netmsb.bat\", \"embedded/.compressor /p:core=\\\"../$(dobj)gnt.bat\\\" /p:output=\\\"../$(dobj)gnt.minified.bat\\\" /nologo /v:m /m:4\", 400)]]", + "#[IO copy.file(\"$(dobj)gnt.bat\", \"$(dobj)gnt.full.bat\", true)]", + "#[IO copy.file(\"$(dobj)gnt.minified.bat\", \"$(dobj)gnt.bat\", true)]", + "#[IO delete.files({\"$(dobj)gnt.minified.bat\"})]", + "", + "#[IO writeLine(STDOUT): Validating the generated version ...]", + "", "#[IO copy.file(\"embedded/sha1*\", \"$(dobj)\", true)]", "#[IO writeLine(STDOUT):#[File sout(\"$(dobj)sha1.cmd\", \"\", 400)]]" ] @@ -130,6 +141,7 @@ "#[IO copy.file(\"$(dobj)core\", \"$(odir)core/gnt.core\", true)]", "#[IO copy.file(\"$(dobj)gnt.min.core\", \"$(odir)core-minified/gnt.core\", true)]", "#[IO copy.file(\"$(dobj)gnt.bat\", \"$(odir)versions/01. executable/gnt.bat\", true)]", + "#[IO copy.file(\"$(dobj)gnt.bat.map\", \"$(odir)versions/01. executable/gnt.bat.map\", true)]", "", "#[IO copy.file({", " \"$(odir)core-minified/gnt.core\", ", diff --git a/README.md b/README.md index 2ae5cb3..eb1632f 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ The [MIT License (MIT)](https://github.com/3F/GetNuTool/blob/master/LICENSE) Copyright (c) 2015-2018 Denis Kuzmin :: github.com/3F ``` -[![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donate_SM.gif)](https://r-eg.net/Donation/) +[![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donate_SM.gif) ☕](https://3F.github.io/Donation/) ## Why GetNuTool ? diff --git a/caller/gnt.bat b/caller/gnt.bat index 99dedba..6c2954b 100644 --- a/caller/gnt.bat +++ b/caller/gnt.bat @@ -1,4 +1,7 @@ @echo off +:: GetNuTool via batch +:: Copyright (c) 2015-2018 Denis Kuzmin [ entry.reg@gmail.com ] +:: https://github.com/3F/GetNuTool set gntcore=gnt.core @@ -11,7 +14,7 @@ for %%v in (4.0, 14.0, 12.0, 3.5, 2.0) do ( ) ) -echo MSBuild was not found, try: ` "full_path_to_msbuild.exe" %gntcore% arguments` 1>&2 +echo MSBuild was not found. Try: "full_path_to_msbuild.exe" %gntcore% arguments` 1>&2 exit /B 2 :found @@ -21,4 +24,4 @@ exit /B 2 %msbuildexe% %gntcore% /nologo /v:m /m:4 %* REM /noconlog -exit /B 0 \ No newline at end of file +exit /B %ERRORLEVEL% \ No newline at end of file diff --git a/embedded/.compressor b/embedded/.compressor new file mode 100644 index 0000000..131e5da --- /dev/null +++ b/embedded/.compressor @@ -0,0 +1,344 @@ + + + + + + + + + gnt.bat + gnt.minified.bat + + + + + + + + + + $(MSBuildToolsPath)\Microsoft.Build.Tasks.v$(MSBuildToolsVersion).dll + $(MSBuildToolsPath)\Microsoft.Build.Tasks.Core.dll + + + + + + + + + + + + + + + , string[]> gencomb = (char[] _dict, int _size, Func _rule0) => + { + var combination = new char[_size]; + var set = new List((int)Math.Pow(_dict.Length, _size)); + + int pos = 0; + Action generator = null; + generator = () => + { + for(int i = 0, lim = _size - 1; i < _dict.Length; ++i) + { + if(pos == 0 && !_rule0(i)) { + continue; + } + + if(pos < lim) { + combination[pos] = _dict[i]; + ++pos; + generator(); + --pos; + } + else { + combination[pos] = _dict[i]; + set.Add(new String(combination.ToArray())); + } + } + }; + + generator(); + return set.ToArray(); + }; + + var variables = new Dictionary(); + + var cdict = new[] { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', + 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', + 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '_' }; + + var vdict = gencomb(cdict, 2, (int i) => { return char.IsLetter(cdict[i]) || cdict[i] == '_'; }); + + // to skip processing for: + var exvar = new[] { "__p_call", "__p_msb" }; + + const string VNAME = "[a-z_][a-z_0-9]+"; + const string VERS = "[Minified version]"; + const string APP = "GetNuTool"; + using(StreamReader reader = new StreamReader(core, System.Text.Encoding.UTF8, true)) + { + var content = reader.ReadToEnd(); + + /* set /a ERROR_ codes */ + + var errorsCodes = new Dictionary(); + content = Regex.Replace + ( + content, + @"set\s+\/a\s+(?'k'ERROR_[^= ]+)\s*=\s*(?'v'\d+)", + (Match m) => + { + errorsCodes[m.Groups["k"].Value] = m.Groups["v"].Value; + return String.Empty; + } + ); + + foreach(var err in errorsCodes) + { + content = Regex.Replace( + content, + String.Format("({1}{0}{1}|{2}{0}{2})", err.Key, "%", "!"), + err.Value + ); + } + + + /* Shorten variables & labels */ + + uint uniqVars = 0; + content = Regex.Replace + ( + content, + @"(?'def' + set\s+? + (?:\/\S\s+?)? + (?:""\s*?)? + ) + (?'name'"+ VNAME + @") + (?'lim'\s?\S?=)", // aq+=1,.. TODO: aq=aq+1 ; aq=1+aq ... + (Match m) => + { + var def = m.Groups["def"].Value; + var vname = m.Groups["name"].Value; + var lim = m.Groups["lim"].Value; + + if(exvar.Contains(vname)) { + return m.Groups[0].Value; + } + + if(variables.ContainsKey(vname)) { + return def + variables[vname] + lim; + } + + if(uniqVars + 1 > vdict.Length) { + throw new OverflowException("vdict does not contain data for new vars"); + } + variables[vname] = vdict[uniqVars++]; + return def + variables[vname] + lim; + }, + RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase + ); + + // call :proc p1 p2 ... + content = Regex.Replace + ( + content, + @"(?'def'call\s+:"+ VNAME + @"\s)(?'args'.+?)(?'eol'&|\r\n)", + (Match m) => + { + var def = m.Groups["def"].Value; + var cargs = m.Groups["args"].Value; + var eol = m.Groups["eol"].Value; + + return def + Regex.Replace + ( + cargs, + @"(?'split'^|\s)(?:(?'str'"".+?"")|(?'name'" + VNAME + "))", + (Match _m) => + { + var split = _m.Groups["split"].Value; + + if(_m.Groups["str"].Success) { + return split + _m.Groups["str"].Value; + } + + var vname = _m.Groups["name"].Value; + + if(exvar.Contains(vname)) { + return split + vname; + } + + if(variables.ContainsKey(vname)) { + return split + variables[vname]; + } + + if(uniqVars + 1 > vdict.Length) { + throw new OverflowException("vdict does not contain data for new vars"); + } + variables[vname] = vdict[uniqVars++]; + return split + variables[vname]; + }, + RegexOptions.IgnoreCase | RegexOptions.Multiline + ) + eol; + }, + RegexOptions.IgnoreCase + ); + + // %name... & !name... + content = Regex.Replace + ( + content, + @"(?'def'(?:[%!]|\sdefined\s+))(?'name'"+ VNAME + ")", + (Match m) => + { + var def = m.Groups["def"].Value; + var vname = m.Groups["name"].Value; + + if(!variables.ContainsKey(vname)) { + return def + vname; + } + return def + variables[vname]; + }, + RegexOptions.IgnoreCase + ); + + // labels + content = Regex.Replace + ( + content, + @"(?'def'call\s*:|^\s*:|goto\s*:?)(?'label'" + VNAME + @")(?'eol'\s|\r\n)", + (Match m) => + { + var def = m.Groups["def"].Value; + var label = m.Groups["label"].Value; + var eol = m.Groups["eol"].Value; + + if(variables.ContainsKey(label)) { + return def + variables[label] + eol; + } + + if(uniqVars + 1 > vdict.Length) { + throw new OverflowException("vdict does not contain data for new labels"); + } + variables[label] = vdict[uniqVars++]; + return def + variables[label] + eol; + }, + RegexOptions.IgnoreCase | RegexOptions.Multiline + ); + + + /* exit/B */ + + content = Regex.Replace(content, @"exit\s*\/B\s*(?'code'\d+)?\s*?", (Match m) => + { + string ret = "exit/B "; + string code = m.Groups["code"].Value; + + if(m.Groups["code"].Success) { + // 'exit /B 0' is not equal to 'exit /B' - it will pass the latest raised code i.e. without changing it. + return ret + code; + } + return ret; + }, + RegexOptions.IgnoreCase); + + + /* pseudo arguments from labels */ + + content = Regex.Replace(content, @"^\s*(?'label':" + VNAME + @").*?\r\n", (Match m) => + { + return m.Groups["label"].Value + "\r\n"; + }, + RegexOptions.IgnoreCase | RegexOptions.Multiline); + + + /* A common rules */ + + //content = Regex.Replace(content, @"(?:rem|::)\s.*[\r\n]*", ""); + + content = Regex.Replace(content, @"(^\s*?(?:rem|::)\s+)(?'data'.*?)\r\n", (Match m) => + { + var data = m.Groups["data"].Value; + + if(!data.StartsWith("Copyright") + && !data.StartsWith("GetNuTool") + && !data.StartsWith("http") + && !data.StartsWith("Based ") + && !data.StartsWith("---")) + { + return String.Empty; + } + + if(data.Contains("/issues")) { + return String.Empty; + } + + return m.Groups[0].Value; + }, + RegexOptions.IgnoreCase | RegexOptions.Multiline); + + content = Regex.Replace(content, @"(\r\n){2,}", "$1"); + content = Regex.Replace(content, @"(\r\n)\s*", "$1"); + content = Regex.Replace(content, @"\s+(\r\n)", "$1"); + + //content = content.Replace(" " + APP + " - ", String.Format(" {0} {1} - ", APP, VERS)); + + using(TextWriter writer = new StreamWriter(output, false, new UTF8Encoding(false))) { + writer.Write(content); + } + Console.WriteLine("{0} version of `{1}` has been created -> `{2}`", VERS, core, output); + } + + + /* map */ + + using(TextWriter wmap = new StreamWriter(core + ".map", false, new UTF8Encoding(false))) + { + string map = String.Empty; + foreach(var v in variables) { + map += String.Format("{0}={1}\n", v.Key, v.Value); + } + + wmap.Write(map); + } + + // using(StreamReader mtpl = new StreamReader(maptpl, System.Text.Encoding.UTF8, true)) + // using(TextWriter wmap = new StreamWriter(core + ".map.targets", false, new UTF8Encoding(false))) + // { + // string map = String.Empty; + // foreach(var v in variables) { + // map += String.Format("\r\n", "$", v.Value, v.Key); + // } + + // wmap.Write(mtpl.ReadToEnd().Replace("{0}", map)); + // } + + ]]> + + + + + + + + \ No newline at end of file diff --git a/embedded/.packer b/embedded/.packer index 63281da..e3eea68 100644 --- a/embedded/.packer +++ b/embedded/.packer @@ -15,7 +15,7 @@ ..\logic.targets gnt.bat - 1577 + 1777 @@ -68,6 +68,28 @@ symbol); }; + Func> scalc = delegate(string content) + { + var data = new Dictionary(); // positions: "(start) -> "(length) + + var matches = Regex.Matches(content, quotes('"'), RegexOptions.IgnorePatternWhitespace); + foreach(Match m in matches) { + data[m.Index] = m.Length; + } + return data; + }; + + Func, bool> isInsideString = delegate(int idx, Dictionary strings) + { + foreach(var s in strings) + { + if(idx > s.Key && idx < (s.Key + s.Value)) { + return true; + } + } + return false; + }; + using(StreamReader reader = new StreamReader(core, System.Text.Encoding.UTF8, true)) { var content = reader.ReadToEnd(); @@ -81,20 +103,18 @@ content = content.Replace("&", "^&"); content = content.Replace("<", "^<"); content = content.Replace(">", "^>"); - content = content.Replace("|", "^|"); + // content = content.Replace("|", "^|"); // - don't escape this inside strings ".." // Required only when enableDelayedExpansion is active //content = content.Replace("!", "^^!"); + // Protecting data outside strings only: + content = Regex.Replace(content, @"\|", (Match m) => isInsideString(m.Index, scalc(content)) ? m.Value : "^" + m.Value); + // Secondly, keep in mind where placed all strings. // We will work without double quotes, so we should correctly define all pairs of "..." per line - var strings = new Dictionary(); // positions: "(start) -> "(end) - - var matches = Regex.Matches(content, quotes('"'), RegexOptions.IgnorePatternWhitespace); - foreach(Match m in matches) { - strings[m.Index] = m.Length; - } + var strings = scalc(content); // Now, we should split long strings @@ -126,15 +146,16 @@ // we don't see strings, but we also have escape chars as ^ + char above - int esc = rpos - 1; // the end of the previous line that should be checked on `^` char - if(content.Length > esc && content.ElementAt(esc) == '^') { + int esc = rpos - 1; // the end of the previous line that should be checked on `^` char + int lchar = Math.Min(rpos, content.Length - 1); + if(content.Length > esc && (content.ElementAt(esc) == '^' || content.ElementAt(lchar) == ' ')) { rlen += 1; lines.Add(content.Substring(i, rlen)); i += 1; continue; } - // TODO: the line cannot be started with: ` =`, `=`, ` "`, `"` + // TODO: the line cannot be started with: ` `(space), `=`, `"` lines.Add(content.Substring(i, rlen)); // simply add new line without processing } @@ -144,9 +165,19 @@ const string corevar = "tgnt"; content = String.Empty; - foreach(var line in lines) { + foreach(var line in lines) + { //content += $"> %{corevar}%{Environment.NewLine}"; - content += String.Format("> %{1}%{2}", line, corevar, Environment.NewLine); + string ldata = String.Format(">%{1}%\r\n", line, corevar); + + if(ldata.Length > 2047) { + Console.WriteLine("[Warn] Length of line {0}", ldata.Length); + } + else if(ldata.Length > 8191) { + Console.Error.WriteLine("[Error] Length of line {0}", ldata.Length); + } + + content += ldata; } // Finally, format script to work with gnt.core @@ -160,7 +191,7 @@ using(TextWriter writer = new StreamWriter(output, false, new UTF8Encoding(false))) { // do not use BOM ! it failed with initial signature writer.Write(content); } - Console.WriteLine("Executable version of `{0}` has been created -> `{1}`", core, output); + Console.WriteLine("Executable version of `{0}` has been created -> `{1}`", core, output); } ]]> diff --git a/embedded/exec.tpl b/embedded/exec.tpl index 80228c8..29f7b85 100644 --- a/embedded/exec.tpl +++ b/embedded/exec.tpl @@ -6,43 +6,88 @@ set gntcore=gnt.core set $tpl.corevar$="%temp%\%random%%random%%gntcore%" -set "args=%* " -set a=%args:~0,30% -set a=%a:"=% +if "%~1"=="-unpack" goto unpack -if "%a:~0,8%"=="-unpack " goto unpack -if "%a:~0,9%"=="-msbuild " goto ufound +set args=%* + +:: Escaping '^' is not identical for all cases (gnt ... vs call gnt ...). +if defined __p_call if defined args set args=%args:^^=^% + +:: When ~ %args% (disableDelayedExpansion) +:: # call gnt ^ - ^^ +:: # gnt ^ - ^ +:: # call gnt ^^ - ^^^^ +:: # gnt ^^ - ^^ + +:: When ~ !args! and "!args!" +:: # call gnt ^ - ^ +:: # gnt ^ - empty +:: # call gnt ^^ - ^^ +:: # gnt ^^ - ^ + +:: Do not use: ~ "%args%" or %args% + (enableDelayedExpansion) + +set msbuildexe=%__p_msb% +if defined msbuildexe goto found + +if "%~1"=="-msbuild" goto ufound for %%v in (4.0, 14.0, 12.0, 3.5, 2.0) do ( for /F "usebackq tokens=2* skip=2" %%a in ( `reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\%%v" /v MSBuildToolsPath 2^> nul` ) do if exist %%b ( - set msbuildexe="%%b\MSBuild.exe" + set msbuildexe="%%~b\MSBuild.exe" goto found ) ) -echo MSBuild was not found, try: gnt -msbuild "fullpath" args 1>&2 +echo MSBuild was not found. Try -msbuild "fullpath" args 1>&2 exit /B 2 :ufound -call :popa %1 shift set msbuildexe=%1 -call :popa %1 +shift + +set esc=%args:!= #__b_ECL## % +setlocal enableDelayedExpansion + +set esc=!esc:%%=%%%%! + +:_eqp +for /F "tokens=1* delims==" %%a in ("!esc!") do ( + if "%%~b"=="" ( + call :nqa !esc! + exit /B %ERRORLEVEL% + ) + set esc=%%a #__b_EQ## %%b +) +goto _eqp +:nqa +shift & shift + +set "args=" +:_ra +set args=!args! %1 +shift & if not "%~2"=="" goto _ra + +set args=!args: #__b_EQ## ==! + +setlocal disableDelayedExpansion +set args=%args: #__b_ECL## =!% :found call :core -%msbuildexe% %$tpl.corevar$% /nologo /p:wpath="%~dp0/" /v:m %args% -del /Q/F %$tpl.corevar$% -exit /B 0 +%msbuildexe% %$tpl.corevar$% /nologo /p:wpath="%~dp0/" /v:m /m:4 %args% -:popa -call set args=%%args:%1 ^=%% -exit /B 0 +set "msbuildexe=" +set ret=%ERRORLEVEL% + +del /Q/F %$tpl.corevar$% +exit /B %ret% :unpack set $tpl.corevar$="%~dp0\%gntcore%" -echo Generate minified version in %$tpl.corevar$% ... +echo Generating minified version in %$tpl.corevar$% ... :core %$tpl.corevar$% diff --git a/embedded/sha1.cmd b/embedded/sha1.cmd index 1ff113b..a9f8e6b 100644 --- a/embedded/sha1.cmd +++ b/embedded/sha1.cmd @@ -12,5 +12,5 @@ exit /B 0 :err -echo Failed. 1>&2 +echo Failed SHA-1. 1>&2 exit /B 1 \ No newline at end of file diff --git a/logic.targets b/logic.targets index 1be0603..4859aff 100644 --- a/logic.targets +++ b/logic.targets @@ -92,7 +92,8 @@ }; var ret = new Queue(); - foreach(var cfg in config.Split('|', ';')) { + foreach(var cfg in config.Split(new char[]{config.IndexOf('|') != -1 ? '|' : ';'}, (StringSplitOptions)1)) + { var lcfg = Path.Combine(wpath, cfg ?? ""); if(File.Exists(lcfg)) { h(lcfg, ret); @@ -106,7 +107,7 @@ _err.WriteLine("List of packages is empty. Use .config or /p:ngpackages=\"...\"\n"); } else { - Result = String.Join(";", ret.ToArray()); + Result = String.Join("|", ret.ToArray()); } ]]> @@ -227,7 +228,7 @@ //Format: id/version[:path];id2/version[:D:/path];... - foreach(var pkg in plist.Split(';')) + foreach(var pkg in plist.Split(new char[]{plist.IndexOf('|') != -1 ? '|' : ';'}, (StringSplitOptions)1)) { var ident = pkg.Split(new char[] { ':' }, 2); var link = ident[0];