From a200982d816b9b24a42a4073624a1994ada3a5a5 Mon Sep 17 00:00:00 2001 From: "Denis Kuzmin [ github.com/3F ]" Date: Sat, 8 Dec 2018 21:08:40 +0300 Subject: [PATCH] New way for arguments passing into main core. And fully updated compression together with .packer for more lightweight distribution, like for DllExport or hMSBuild project. // TODO: code review and fully new structure! this is just a working draft version of the .packer and its compressors. But now it's really complex pre/post-processing than before o_o --- README.md | 38 ++++--- embedded/.packer | 170 ++++++++++++++++++++++++++--- logic.targets | 60 +++++------ minified/.compressor | 247 ++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 441 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index eb1632f..2352ce5 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Initially this was developed for providing tools to service projects, libraries, * [Conari](https://github.com/3F/Conari) * [DllExport](https://github.com/3F/DllExport) +* [MvsSln](https://github.com/3F/MvsSln) * [vsSolutionBuildEvent](https://github.com/3F/vsSolutionBuildEvent) * [LunaRoad](https://github.com/3F/LunaRoad) * [vsCommandEvent](https://github.com/3F/vsCommandEvent) @@ -115,13 +116,24 @@ gnt /p:ngpackages="Conari" /p:proxycfg="guest:1234@10.0.2.15:7428" #### Format of packages list -Attribute | Description +``` +id[/version][:output]|id2[/version][:output]|... +``` + +Attribute | Description | Example +----------|---------------------------------------------|------------------------------ +id | Identifier of package. | [Conari](https://www.nuget.org/packages/Conari) +version | **(Optional)** Version of package. | `1.3.0` or `1.3-beta2` or `1.3-RC` etc +output | **(Optional)** Path to write package data. | `../tests/ConariForTest` + +Multiple packages: + +Delimiter | Description ----------|------------- -id | Identifier of package. -version | Version of package. -output | Optional path for getting package. + `;` | `id1;id2;id3` Optional, starting from 1.7+. Means usage as an delimiter **only when** the `|` is not found. + `|` | `id1:dir;name|id2` Package **id1** into directory **dir;name** and package **id2** in `ngpath` path. -Property: +To use via arguments: ```bash /p:ngpackages="id[/version][:output]" @@ -131,7 +143,7 @@ Property: /p:ngpackages="id[/version][:output];id2[/version][:output];..." ``` -packages.config: +To use via packages.config: ```xml @@ -148,10 +160,10 @@ packages.config: /p:ngconfig=".nuget/packages.config" ``` -multiple: +Multiple config files via delimiters: -* `;` - v1.6+ -* `|` - v1.0+ (*obsolete and can be removed in new versions*) +* `;` - v1.6+ (Optional, starting from 1.7+. Means usage as an delimiter **only when** the `|` is not found.) +* `|` - v1.0+ ```bash /p:ngconfig="debug.config;release.config;..." @@ -253,7 +265,7 @@ Now, you can use it simply: ### Additional arguments -key | Description | Sample -----------------|---------------------------------------------------------|---------------- -`-unpack` | To generate minified version from executable. `v1.6+` | `gnt -unpack` -`-msbuild` path | To use specific msbuild if needed. `v1.6+` | `gnt -msbuild "D:\MSBuild\bin\amd64\msbuild" /p:ngpackages="Conari"` \ No newline at end of file + First key to gnt | Description | Sample +------------------|---------------------------------------------------------|---------------- + `-unpack` | To generate minified version from executable. `v1.6+` | `gnt -unpack` + `-msbuild` path | To use specific msbuild if needed. `v1.6+` | `gnt -msbuild "D:\MSBuild\bin\amd64\msbuild" /p:ngpackages="Conari"` \ No newline at end of file diff --git a/embedded/.packer b/embedded/.packer index e3eea68..bd104a8 100644 --- a/embedded/.packer +++ b/embedded/.packer @@ -15,7 +15,7 @@ ..\logic.targets gnt.bat - 1777 + 1820 @@ -90,6 +90,60 @@ return false; }; + Func, string[]> gencomb = (char[] _dict, int _size, Func _rule0) => + { + var combination = new char[_size]; + var set = new List((int)Math.Pow(_dict.Length, _size)); + + int pos = 0; + Action generator = null; + generator = () => + { + for(int i = 0, lim = _size - 1; i < _dict.Length; ++i) + { + if(pos == 0 && !_rule0(i)) { + continue; + } + + if(pos < lim) { + combination[pos] = _dict[i]; + ++pos; + generator(); + --pos; + } + else { + combination[pos] = _dict[i]; + set.Add(new String(combination.ToArray())); + } + } + }; + + generator(); + return set.ToArray(); + }; + + var variables = new Dictionary(); + + var cdict = new[] { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', + 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', + 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '_' }; + + var vdict = gencomb(cdict, 1, (int i) => { return char.IsLetter(cdict[i]) || cdict[i] == '_'; }); + + // to skip processing for: + var exword = new[] { "" }; + int skipFirst = 110; + + const string rlexemes = @"(?: + (?'wrd' + Console\s*\.\s*WriteLine + ) + | + (?'ld'%%?0[aA])? + (?'cls'[a-z0-9A-Z_\-]+) + )"; + using(StreamReader reader = new StreamReader(core, System.Text.Encoding.UTF8, true)) { var content = reader.ReadToEnd(); @@ -109,16 +163,103 @@ //content = content.Replace("!", "^^!"); // Protecting data outside strings only: - content = Regex.Replace(content, @"\|", (Match m) => isInsideString(m.Index, scalc(content)) ? m.Value : "^" + m.Value); + content = Regex.Replace(content, @"\|", (m) => isInsideString(m.Index, scalc(content)) ? m.Value : "^" + m.Value); + + // lexemes via vars + + var lstat = new Dictionary(); + foreach(Match m in Regex.Matches(content, rlexemes, RegexOptions.IgnorePatternWhitespace)) + { + var wrd = m.Groups["wrd"]; + var cls = m.Groups["cls"]; + + var vname = wrd.Success ? wrd.Value : cls.Value; + + if(vname.Length < 4) { // minimal is 3: %a% + continue; + } - // Secondly, keep in mind where placed all strings. - // We will work without double quotes, so we should correctly define all pairs of "..." per line + if(exword.Contains(vname)) { + continue; + } + + if(!lstat.ContainsKey(vname)) { + lstat[vname] = -1 * (vname.Length + 7); //for init: set a= + length + & + } + lstat[vname] += (vname.Length - 3); //for use: %a% + } + + uint uniqVars = 0; + variables.Clear(); + var prio = lstat.Where(x => x.Value > 0).OrderByDescending(x => x.Value).Take(vdict.Length); + var defv = ""; + + content = Regex.Replace(content, rlexemes, (m) => + { + if(m.Index < skipFirst) { + return m.Value; + } + + var wrd = m.Groups["wrd"]; + var cls = m.Groups["cls"]; + var ld = m.Groups["ld"].Value; + + var vname = wrd.Success ? wrd.Value : cls.Value; + + if(prio.All(x => x.Key != vname)) { + return m.Value; + } + + if(variables.ContainsKey(vname)) { + return ld + variables[vname]; + } + + if(uniqVars + 1 > vdict.Length) { + throw new OverflowException("vdict does not contain data for new vars"); + } + + var nv = vdict[uniqVars++]; + + variables[vname] = String.Format("%{0}%", nv); + + defv += String.Format("set {0}={1}&", nv, vname); + return ld + variables[vname]; + }, + RegexOptions.IgnorePatternWhitespace); + + + // We'll work without double quotes, so we should define all pairs of "..." per line var strings = scalc(content); // Now, we should split long strings var lines = new List(); + + // Protecting from some symbols in zero position via moving to the end of the previous line + Action safeAdd = delegate(int idx, int l) + { + int lofs = idx; + while( (content[lofs] == ' ' || content[lofs] == ')' || content[lofs] == '=') + || + //TODO: lexemes via vars %a%, %ab%, ... + escaped %%data + (content[lofs] == '%' || content[Math.Min(lofs + 1, content.Length - 1)] == '%') + ) + { + ++lofs; + } + + int ofsdelta = lofs - idx; + if(ofsdelta > 0) + { + lines[lines.Count - 1] += content.Substring(idx, ofsdelta); + lines.Add(content.Substring(lofs, l - ofsdelta)); + return; + } + + lines.Add(content.Substring(idx, l)); + }; + for(int i = 0, len = maxline; i < content.Length; i += len) { int rlen = Math.Min(content.Length - i, len); @@ -132,7 +273,7 @@ { int repos = ((ifq.Key + ifq.Value) - rpos); - lines.Add(content.Substring(i, rlen + repos)); + safeAdd(i, rlen + repos); i += repos; added = true; @@ -144,20 +285,17 @@ continue; } - // we don't see strings, but we also have escape chars as ^ + char above + // Protecting from escaped chars as ^ + char above - int esc = rpos - 1; // the end of the previous line that should be checked on `^` char - int lchar = Math.Min(rpos, content.Length - 1); - if(content.Length > esc && (content.ElementAt(esc) == '^' || content.ElementAt(lchar) == ' ')) { + int esc = rpos - 1; // the end of the previous line that should be checked on `^` char + if(content.Length > esc && content[esc] == '^') { rlen += 1; - lines.Add(content.Substring(i, rlen)); + safeAdd(i, rlen); i += 1; continue; } - - // TODO: the line cannot be started with: ` `(space), `=`, `"` - - lines.Add(content.Substring(i, rlen)); // simply add new line without processing + + safeAdd(i, rlen); } // Now we can define, how it should be written into external file @@ -180,6 +318,10 @@ content += ldata; } + // definition of lexemes + + content = defv.Substring(0, defv.Length - 1) + "\r\n" + content; + // Finally, format script to work with gnt.core using(StreamReader tplreader = new StreamReader("exec.tpl", System.Text.Encoding.UTF8, true)) { diff --git a/logic.targets b/logic.targets index 4859aff..7b4db2d 100644 --- a/logic.targets +++ b/logic.targets @@ -25,13 +25,13 @@ - + - + - + @@ -44,9 +44,6 @@ - - - @@ -59,6 +56,10 @@ (); foreach(var cfg in config.Split(new char[]{config.IndexOf('|') != -1 ? '|' : ';'}, (StringSplitOptions)1)) { - var lcfg = Path.Combine(wpath, cfg ?? ""); + var lcfg = Path.Combine(wpath, cfg); if(File.Exists(lcfg)) { h(lcfg, ret); } @@ -104,7 +105,7 @@ } if(ret.Count < 1) { - _err.WriteLine("List of packages is empty. Use .config or /p:ngpackages=\"...\"\n"); + _err.WriteLine("Empty list. Use .config or /p:ngpackages=\"...\"\n"); } else { Result = String.Join("|", ret.ToArray()); @@ -116,16 +117,9 @@ - - - - - - - @@ -133,6 +127,12 @@ loc = delegate(string p) { - return Path.Combine(wpath, p ?? ""); + return Path.Combine(wpath, p ?? String.Empty); }; Action get = delegate(string link, string name, string path) @@ -182,10 +182,10 @@ { try { - if(proxy != null) { + if(!String.IsNullOrEmpty(proxy)) { l.Proxy = getProxy(proxy); } - l.Headers.Add("User-Agent", "GetNuTool"); + l.Headers.Add("User-Agent", "GetNuTool $(GetNuTool)"); l.UseDefaultCredentials = true; l.DownloadFile(url + link, tmp); } @@ -247,15 +247,6 @@ - - - - - - - - - @@ -269,6 +260,11 @@ getmeta = delegate(string key) { - return (metadata.ContainsKey(key))? metadata[key] : ""; + return (metadata.ContainsKey(key))? metadata[key] : String.Empty; }; var _p = pkg.PackageProperties; @@ -390,7 +386,7 @@ _p.Version = metadata[VER]; _p.Keywords = getmeta("tags"); _p.Title = getmeta("title"); - _p.LastModifiedBy = "GetNuTool v" + vtool; + _p.LastModifiedBy = "GetNuTool $(GetNuTool)"; } ]]> @@ -406,7 +402,7 @@ $(MSBuildProjectDirectory) - + \ No newline at end of file diff --git a/minified/.compressor b/minified/.compressor index adbe404..6a67ec4 100644 --- a/minified/.compressor +++ b/minified/.compressor @@ -43,6 +43,7 @@ + , string[]> gencomb = (char[] _dict, int _size, Func _rule0) => + { + var combination = new char[_size]; + var set = new List((int)Math.Pow(_dict.Length, _size)); + + int pos = 0; + Action generator = null; + generator = () => + { + for(int i = 0, lim = _size - 1; i < _dict.Length; ++i) + { + if(pos == 0 && !_rule0(i)) { + continue; + } + + if(pos < lim) { + combination[pos] = _dict[i]; + ++pos; + generator(); + --pos; + } + else { + combination[pos] = _dict[i]; + set.Add(new String(combination.ToArray())); + } + } + }; + + generator(); + return set.ToArray(); + }; + + var variables = new Dictionary(); + + var cdict = new[] { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', + 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', + 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '_' }; + + Func rule = (int i) => { return char.IsLetter(cdict[i]) || cdict[i] == '_'; }; + var vdict1 = gencomb(cdict, 1, rule); + var vdict2 = gencomb(cdict, 2, rule); + + var vdict = new string[vdict1.Length + vdict2.Length]; + vdict1.CopyTo(vdict, 0); + vdict2.CopyTo(vdict, vdict1.Length); + + // to skip processing for: + var exvar = new string[] { "true", "false" }; + + // https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/keywords/ + var rsvwords = new[] { "do", "in", "is", "as", "if", "by", "on" }; + + const string VTYPE = @"(?: + (?'type' + [a-z_] + (?: + [a-z_0-9]+<[a-z_0-9<, >]+> + | + [a-z_0-9]+ + ) + ) + \s+ + (?'name'[a-z_0-9]+) + \s* + (?'lim' + (?:[+\-*\/]?=|\s+in\s+|[,)]|\s*;) + ) + | + (?: + (?'fl'\(\s*\(\s*)(?'flambda'[^)]+?)\)?\s* + | + (?'vl'\(\s*)(?'vlambda'[^=\s])\s* + )=> + ) + "; + + const string VNAME = "[a-z_0-9]+"; // [a-z_][a-z_0-9]+ + + using(StreamReader reader = new StreamReader(core, System.Text.Encoding.UTF8, true)) { var content = reader.ReadToEnd(); @@ -96,29 +177,134 @@ return String.Format("!s{0}!", ident); }, RegexOptions.IgnorePatternWhitespace); + + // code + + content = Regex.Replace(content, @"", (Match m) => + { + var data = m.Groups[0].Value; - // C# code + // comments - content = Regex.Replace(content, - @"", - delegate(Match m) - { - var data = m.Groups[0].Value; + data = Regex.Replace(data, @"\s*\/\*.+?\*\/\s*", "", RegexOptions.Singleline); + data = Regex.Replace(data, @"\s*\/\/.+?$", "", RegexOptions.Multiline); + + // shorten variables + // TODO: sorting by popularity of usage for 1 byte priority + + variables.Clear(); + uint uniqVars = 0; + + Func shname = delegate(Match _m, string l, string vname, string r) + { + if(!Regex.IsMatch(vname, "^[a-z_]")) { + return _m.Value; + } - data = Regex.Replace(data, @"\s*\/\*.+?\*\/\s*", "", RegexOptions.Singleline); - data = Regex.Replace(data, @"\s*//[^$]+?$", "", RegexOptions.Multiline); - data = Regex.Replace(data, @"\s+const\s+[sS]tring\s+", " var "); - data = Regex.Replace(data, @"([;(){}]\s*)\w+\s+(\w+)\s+(in|=)\s+", " $1 var $2 $3 "); - data = Regex.Replace(data, @"\s*([{}()=+\-\[\]*?!@,;.])\s*", "$1"); + if(exvar.Contains(vname)) { + return _m.Value; + } - return data; - }, - RegexOptions.Singleline); + if(variables.ContainsKey(vname)) { + return l + variables[vname] + r; + } + + if(uniqVars + 1 > vdict.Length) { + throw new OverflowException("vdict does not contain data for new vars"); + } + + do { + variables[vname] = vdict[uniqVars++]; + } + while(rsvwords.Contains(variables[vname])); + + return l + variables[vname] + r; + }; + + data = Regex.Replace + ( + data, + VTYPE, + (Match _m) => + { + var vname = _m.Groups["name"]; + + if(vname.Success) { + return shname(_m, _m.Groups["type"].Value + " ", vname.Value, _m.Groups["lim"].Value); + } + + var flambda = _m.Groups["flambda"]; + if(flambda.Success) { + return Regex.Replace(flambda.Value, VNAME, __m => shname(__m, _m.Groups["fl"].Value, __m.Value, "=>")); + } + + var vlambda = _m.Groups["vlambda"]; + if(vlambda.Success) { + return Regex.Replace(vlambda.Value, VNAME, __m => shname(__m, _m.Groups["vl"].Value, __m.Value, "=>")); + } + + return _m.Value; + }, + RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase + ); + + // using of variables + data = Regex.Replace + ( + data, + @"(?'def'^|\s+in\s+|(?<==)>|[={};:\[(,?+\-]) + \s* + (?'name'"+ VNAME + @") + (?'exc'[<>]*)? + ", + (Match _m) => + { + var def = _m.Groups["def"].Value; + var vname = _m.Groups["name"].Value; + var exc = _m.Groups["exc"].Value; + + if(exc.IndexOfAny(new[] { '<', '>' }) != -1) { + return _m.Value; + } + + if(!Regex.IsMatch(vname, "^[a-z_]")) { + return _m.Value; + } + + if(!variables.ContainsKey(vname)) { + return def + vname + exc; + } + return def + variables[vname] + exc; + }, + RegexOptions.IgnorePatternWhitespace | RegexOptions.IgnoreCase | RegexOptions.Multiline + ); + + // CDATA rules + + data = Regex.Replace(data, @"([\s;},])String([\s.(]+)", "$1string$2"); + data = Regex.Replace(data, @"[sS]tring\s*\.\s*Empty", "\"\""); + data = Regex.Replace(data, @"\s*const\s+[sS]tring\s+", "var "); + data = Regex.Replace(data, @"(? + { + var def = _m.Groups["def"].Value; + if(def.IndexOf('=') == -1) { + return _m.Value; + } + return "var " + def; + }); + + data = Regex.Replace(data, @"([;(){}]\s*)\w+\s+(\w+)\s+(in|=)\s+", " $1 var $2 $3 "); + data = Regex.Replace(data, @"\s*([{}()=+\-\[\]*?!@,;.])\s*", "$1"); + + return data; + }, + RegexOptions.Singleline); // common rules content = content.Replace("\r", "").Replace("\n", ""); content = Regex.Replace(content, @"\s{2,}", " "); + content = Regex.Replace(content, @"\s*([=,()\[\];:.&|{}\/<>]+)\s*", "$1"); // XML rules @@ -129,10 +315,41 @@ // recover strings - content = Regex.Replace(content, @"!s(\d+)!", delegate (Match m) { + content = Regex.Replace(content, @"!s(\d+)!", (Match m) => { return strings[uint.Parse(m.Groups[1].Value)]; }); + // xml Tasknames + + variables.Clear(); + uint uniqt = 0; + content = Regex.Replace(content, @"TaskName\s*=\s*""(?'name'\S+)""", (_m) => + { + var tname = _m.Groups["name"].Value; + + variables[tname] = vdict[uniqt++]; + + return "TaskName=\"" + variables[tname] + "\""; + }); + + content = Regex.Replace(content, @"(?'l'<\/?)(?'name'[a-z0-9A-Z\-_]+)", (_m) => + { + var tname = _m.Groups["name"].Value; + + if(variables.ContainsKey(tname)) { + return _m.Groups["l"].Value + variables[tname]; + } + + return _m.Value; + }); + + // XML rules - Post filter + + content = Regex.Replace(content, @"\s+ParameterType\s*=\s*""System.String""\s*", " "); + content = Regex.Replace(content, @"\s+Required\s*=\s*""\S+""\s*", " "); + content = Regex.Replace(content, @"\s+(\/)?>", "$1>"); + content = Regex.Replace(content, @"'\s*==\s*'", "'=='"); + using(TextWriter writer = new StreamWriter(output, false, new UTF8Encoding(false))) { writer.Write(content); }