-
Notifications
You must be signed in to change notification settings - Fork 1
/
CutPlugin.cs
81 lines (64 loc) · 2.76 KB
/
CutPlugin.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
namespace Metacrack
{
public class CutPlugin : PluginBase
{
public static void Process(CutOptions options)
{
var currentDirectory = Directory.GetCurrentDirectory();
var fileEntries = Directory.GetFiles(currentDirectory, options.InputPath);
if (fileEntries.Length == 0)
{
WriteMessage($"File(s) {options.InputPath} not found.");
return;
}
if (options.Start >= options.End )
{
WriteMessage("End value must be greater than start");
return;
}
WriteMessage($"Cutting new file {options.OutputPath} from line {options.Start} to line {options.End}");
var size = GetFileEntriesSize(fileEntries);
var progressTotal = 0L;
foreach (var filePath in fileEntries)
{
var output = new List<string>();
var fileInfo = new FileInfo(filePath);
var fileName = Path.GetFileNameWithoutExtension(filePath);
var filePathName = $"{currentDirectory}\\{fileName}";
var outputPath = $"{currentDirectory}\\{options.OutputPath}";
//Check that there are no output files
if (!CheckForFiles(new string[] { outputPath }))
{
WriteHighlight($"Skipping {filePathName}.");
progressTotal += fileInfo.Length;
continue;
}
var lineCount = 0;
//Loop through and check if each email contains items from the lookup, if so add them
using (var reader = new StreamReader(filePath))
{
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
lineCount++;
progressTotal += line.Length + 2;
//We start from line 1
if (lineCount >= options.Start && lineCount <= options.End) output.Add(line);
//Update the percentage
if (lineCount % 1000 == 0) WriteProgress($"Reading {fileName}", progressTotal, size);
//Write out buffer
if (output.Count > 1000000)
{
File.AppendAllLines(outputPath, output);
output.Clear();
}
//Break out early
if (lineCount > options.End) break;
}
}
//Check if we must deduplicate
File.AppendAllLines(outputPath, output);
}
}
}
}