-
Notifications
You must be signed in to change notification settings - Fork 0
/
TestStateless.java
144 lines (140 loc) · 4.58 KB
/
TestStateless.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
import java.util.*; import java.io.*; public class TestStateless{
public static void main(String args[]){
ArrayList<String> deleteMe = new ArrayList<>();
try{
if(args.length!=5){
System.out.println("Usage: java TestStateless mapperFile reducerFile inputFile outputFile deleteTempFiles");
System.exit(0);
}
//split up the input
BufferedReader file = new BufferedReader(new FileReader(args[2]));
int fileNumber = 0;
String line = file.readLine();
while(line!=null){
PrintWriter fout = new PrintWriter(new FileWriter("mapperOut"+fileNumber+".tmp"));
fout.println(line);
fout.close();
deleteMe.add("mapperOut"+fileNumber+".tmp");
fileNumber++;
line = file.readLine();
}
file.close();
String delimeter = "\t";
ArrayList<String> everything = new ArrayList<>();
//run the mapper on each line of the input
for(int i=0; i<fileNumber; i++){
ProcessBuilder builder = new ProcessBuilder();
builder.command("python3", args[0]);
Process p = builder.start();
OutputStream pos = p.getOutputStream();
InputStream ios = p.getInputStream();
InputStream fis = new FileInputStream("mapperOut"+i+".tmp");
byte[] buffer = new byte[1024];
int read = 0;
while((read = fis.read(buffer)) != -1) {
pos.write(buffer, 0, read);
}
pos.close();
fis.close();
p.waitFor();
BufferedReader buff = new BufferedReader(new InputStreamReader(ios));
line = buff.readLine();
while(line!=null){
everything.add(line);
line = buff.readLine();
}
}
Collections.sort(everything);
ArrayList<String> tempEverything = new ArrayList<String>();
String curKey = "";
int beginning = 0;
for(int end=0; end<everything.size(); end++){
String now = everything.get(end).substring(0, everything.get(end).indexOf(delimeter));
if(curKey.equals("")){
//beginning of list, get first key
curKey = now;
} else if(!curKey.equals(now)){
//got a new key, shuffle from beginning to end-1 so the values aren't sorted
List<String> tempSort = everything.subList(beginning, end);
Collections.shuffle(tempSort);
tempEverything.addAll(tempSort);
//store new key and restart the numbering
curKey = now;
beginning = end;
}
}
//add the last key to the mix
List<String> tempSort = everything.subList(beginning, everything.size());
Collections.shuffle(tempSort);
tempEverything.addAll(tempSort);
everything = tempEverything;
//all keys are sorted but the values are not
fileNumber = 0;
String curString = "";
curKey = "";
for(String s : everything){
String key = s.substring(0, s.indexOf(delimeter));
String value = s.substring(s.indexOf(delimeter)+1, s.length());
if(curKey.equals("")){
//it's the start
curKey = key;
curString = s;
} else{
if(curKey.equals(key)){
curString += "\n"+s;
}else{
//got a new key, output old stuff
PrintWriter fout = new PrintWriter(new FileWriter("reducerIn"+fileNumber+".tmp"));
fout.print(curString);
fout.close();
deleteMe.add("reducerIn"+fileNumber+".tmp");
curKey = key;
curString = s;
fileNumber++;
}
}
}
//output the last key
PrintWriter fout = new PrintWriter(new FileWriter("reducerIn"+fileNumber+".tmp"));
fout.print(curString);
deleteMe.add("reducerIn"+fileNumber+".tmp");
fileNumber++;
fout.close();
//run reducer on them
PrintWriter finalOutput = new PrintWriter(new FileWriter(args[3]));
for(int i=0; i<fileNumber; i++){
ProcessBuilder builder = new ProcessBuilder();
builder.command("python3", args[1]);
Process p = builder.start();
OutputStream pos = p.getOutputStream();
InputStream ios = p.getInputStream();
InputStream fis = new FileInputStream("reducerIn"+i+".tmp");
byte[] buffer = new byte[1024];
int read = 0;
while((read = fis.read(buffer)) != -1) {
pos.write(buffer, 0, read);
}
pos.close();
fis.close();
p.waitFor();
BufferedReader buff = new BufferedReader(new InputStreamReader(ios));
line = buff.readLine();
while(line!=null){
finalOutput.println(line);
line = buff.readLine();
}
}
finalOutput.close();
} catch(Exception e){
//something went wrong
System.out.println(e);
} finally{
if(args[4].equalsIgnoreCase("N")){
for(String gone : deleteMe){
File f = new File(gone);
f.delete();
}
}
}
}
}