Skip to content

Commit d728dea

Browse files
MorikkaMorikka
committed
Initial commit
0 parents  commit d728dea

File tree

13 files changed

+698
-0
lines changed

13 files changed

+698
-0
lines changed

‎.gitignore

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
.DS_Store
2+
*.class
3+
*.jar
4+
TESTING/*
5+
test*
6+
tmp*
7+
*.cc
8+
Latex/*

‎CRT.java

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
import java.util.*;
2+
import java.lang.*;
3+
import java.io.*;
4+
5+
import org.apache.hadoop.io.IntWritable;
6+
import org.apache.hadoop.io.LongWritable;
7+
import org.apache.hadoop.io.Text;
8+
9+
import org.apache.logging.log4j.Logger;
10+
import org.apache.logging.log4j.LogManager;
11+
12+
import org.apache.hadoop.conf.Configuration;
13+
import org.apache.hadoop.fs.Path;
14+
import org.apache.hadoop.io.IntWritable;
15+
import org.apache.hadoop.io.Text;
16+
import org.apache.hadoop.mapreduce.Job;
17+
import org.apache.hadoop.mapreduce.Mapper;
18+
import org.apache.hadoop.mapreduce.Reducer;
19+
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
20+
// import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
21+
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
22+
import org.apache.hadoop.mapreduce.lib.chain.ChainMapper;
23+
// import org.apache.hadoop.mapred.JobConf;
24+
import org.apache.hadoop.fs.FileSystem;
25+
// import java.io.OutputStreamWriter;
26+
import org.apache.hadoop.conf.Configured;
27+
import org.apache.hadoop.util.Tool;
28+
import org.apache.hadoop.util.ToolRunner;
29+
30+
public class CRT extends Configured implements Tool {
31+
static Boolean OPTION;
32+
// static final int[] PRIMES = {2,3,5,7,11,13,17,19,23,29,31,37};
33+
static final int[] PRIMES = {23,29,31,37,41,43,47,53,59,61,67};
34+
static final int NUMBER = 11;
35+
static int FAILED_NUMBER;
36+
static int x1;
37+
static int x2;
38+
39+
static final Logger logger = LogManager.getLogger();
40+
static Configuration conf = new Configuration();
41+
42+
public static void addProperty() throws IOException{
43+
Properties prop = new Properties();
44+
InputStream input = null;
45+
input = new FileInputStream("config.properties");
46+
// load a properties file
47+
prop.load(input);
48+
// get the property value and print it out
49+
OPTION = Boolean.valueOf(prop.getProperty("OPTION"));
50+
FAILED_NUMBER = Integer.valueOf(prop.getProperty("FAILED_NUMBER"));
51+
long mx = 1;
52+
for(int i=0;i<PRIMES.length;i++)
53+
mx = mx * PRIMES[i];
54+
long tmp = (long)(Math.random()*50)+1;
55+
x1 = (int)(Math.sqrt(mx)-tmp);
56+
tmp = (long)(Math.random()*50)+1;
57+
x2 = (int)(Math.sqrt(mx)-tmp);
58+
// System.out.println("X1 and X2 is: "+x1+" "+x2);
59+
System.out.println("The First Number="+x1);
60+
System.out.println("The Second Number="+x2);
61+
// System.out.println("X1 and X2 is: "+x1+" "+x2);
62+
}
63+
64+
private static int[] failNode(){
65+
int n = NUMBER;
66+
int[] ans = new int[n];
67+
int[] rand_number = new int[n];
68+
Random rand = new Random();
69+
for(int i=0;i<n;i++){
70+
rand_number[i] = i;
71+
ans[i] = 0;
72+
}
73+
for(int i=0;i<FAILED_NUMBER;i++){
74+
int pos = rand.nextInt(n-i);
75+
// swap(rand[n-i],rand[pos]);
76+
int tmp = rand_number[pos];
77+
rand_number[pos] = rand_number[n-i-1];
78+
rand_number[n-i-1] = tmp;
79+
ans[tmp] = 1;
80+
}
81+
return ans;
82+
}
83+
84+
public static void init() throws IOException {
85+
int[] fail_node = failNode();
86+
int LEN = PRIMES.length;
87+
int[][] matrixA = new int[NUMBER][LEN];
88+
// If with coding, the maximal mapper number equals to number, otherwise the maximal mapper number equals to the prime's number
89+
int max_num = OPTION ? NUMBER : LEN;
90+
91+
if(OPTION){
92+
// CRTMatrixA mtx = new CRTMatrixA(LEN,NUMBER,FAILED_NUMBER,PRIMES);
93+
CRTMatrixD mtx = new CRTMatrixD(LEN,NUMBER,FAILED_NUMBER,PRIMES);
94+
matrixA = mtx.run();
95+
}
96+
97+
String tmpoutput="hdfs://localhost:9000/user/mio/crt/input/data";
98+
String output;
99+
FileSystem Phdfs = FileSystem.get(new Configuration());
100+
String fileStr = "";
101+
102+
for(int i=0;i<max_num;i++){
103+
fileStr = (fail_node[i]==1) ? "1 " : "0 "; // 1 means the node will fail
104+
if(OPTION){
105+
int multi = 1;
106+
String tstr = "";
107+
for(int j=0;j<LEN;j++){
108+
if(matrixA[i][j]==1){
109+
multi *= PRIMES[j];
110+
tstr += String.valueOf(PRIMES[j])+" ";
111+
}
112+
}
113+
conf.set(String.valueOf(multi),tstr);
114+
fileStr += String.valueOf(multi)+" "+String.valueOf(x1)+" "+String.valueOf(x2);
115+
System.out.println("Config: "+tstr);
116+
} else {
117+
fileStr += String.valueOf(PRIMES[i])+" "+String.valueOf(x1)+" "+String.valueOf(x2);
118+
}
119+
output = tmpoutput + String.valueOf(i) +".txt";
120+
Path fname=new Path(output);
121+
BufferedWriter out=new BufferedWriter(new OutputStreamWriter(Phdfs.create(fname,true)));
122+
System.out.println(fileStr);
123+
out.write(fileStr);
124+
out.close();
125+
}
126+
}
127+
128+
@Override
129+
public int run(String[] args) throws Exception {
130+
Job job = Job.getInstance(conf, "crt");
131+
job.setJarByClass(CRT.class);
132+
133+
job.setMapperClass(CRTMapper.class);
134+
job.setMapOutputKeyClass(Text.class);
135+
job.setMapOutputValueClass(IntWritable.class);
136+
137+
job.setReducerClass(CRTReducer.class);
138+
job.setNumReduceTasks(1);
139+
job.setOutputKeyClass(Text.class);
140+
job.setOutputValueClass(LongWritable.class);
141+
142+
TextInputFormat.addInputPath(job, new Path(args[0]));
143+
FileOutputFormat.setOutputPath(job, new Path(args[1]));
144+
145+
return job.waitForCompletion(true) ? 0 : 1;
146+
}
147+
148+
public static void main(String[] args) throws Exception {
149+
addProperty();
150+
conf.setBoolean("OPTION",OPTION);
151+
System.out.println("Config is: "+OPTION+" "+FAILED_NUMBER);
152+
init();
153+
long start = new Date().getTime();
154+
int exitCode = ToolRunner.run(new CRT(),args);
155+
long end = new Date().getTime();
156+
System.out.println("Whole time="+(end-start));
157+
System.exit(exitCode);
158+
}
159+
}

‎CRTMapper.java

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
import java.lang.Math;
2+
3+
import java.io.IOException;
4+
5+
import org.apache.hadoop.io.IntWritable;
6+
import org.apache.hadoop.io.LongWritable;
7+
import org.apache.hadoop.io.NullWritable;
8+
9+
import org.apache.hadoop.io.Text;
10+
import org.apache.hadoop.mapreduce.Mapper;
11+
import org.apache.hadoop.conf.Configuration;
12+
import java.util.StringTokenizer;
13+
import org.apache.hadoop.mapreduce.TaskAttemptID;
14+
import org.apache.hadoop.mapreduce.MRJobConfig;
15+
import org.apache.hadoop.mapreduce.Job;
16+
import org.apache.hadoop.conf.Configuration;
17+
import org.apache.hadoop.mapreduce.Cluster;
18+
19+
public class CRTMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
20+
21+
private Text remainer_key = new Text();
22+
private int dividend,divisor;
23+
private long remainer;
24+
private Integer d;
25+
private Configuration conf;
26+
private Boolean option;
27+
28+
@Override
29+
public void run(Context context) throws IOException, InterruptedException {
30+
conf = context.getConfiguration();
31+
option = conf.getBoolean("OPTION",true);
32+
setup(context);
33+
try {
34+
while (context.nextKeyValue()) {
35+
map(context.getCurrentKey(), context.getCurrentValue(), context);
36+
}
37+
} finally {
38+
cleanup(context);
39+
}
40+
}
41+
42+
@Override
43+
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
44+
System.out.println("The map key: "+key+" value: "+value);
45+
StringTokenizer itr = new StringTokenizer(value.toString());
46+
d = Integer.parseInt(itr.nextToken());
47+
Integer tmp = Integer.parseInt(itr.nextToken());
48+
divisor = tmp.intValue();
49+
remainer = 1;
50+
remainer_key.set(String.valueOf(divisor));
51+
while (itr.hasMoreTokens()) {
52+
tmp = Integer.parseInt(itr.nextToken());
53+
dividend = tmp.intValue();
54+
remainer = remainer * (dividend % divisor) % divisor;
55+
}
56+
}
57+
@Override
58+
protected void cleanup(Context context) throws IOException, InterruptedException {
59+
Cluster cluster = new Cluster(conf);
60+
Job job = cluster.getJob(context.getJobID());
61+
TaskAttemptID tid = context.getTaskAttemptID();
62+
63+
if(option){
64+
if(d.intValue()==0){
65+
context.write(remainer_key, new IntWritable((int)remainer));
66+
System.out.println("Output: "+remainer_key+" "+remainer);
67+
}
68+
} else {
69+
Boolean task = false;
70+
String tmp = tid.toString();
71+
if(d.intValue()==1 && tmp.substring(tmp.length()-1).equals("0")){
72+
System.out.println(tid);
73+
System.out.println("Failed");
74+
job.failTask(tid);
75+
} else {
76+
System.out.println("The map print:" + remainer_key + " "+remainer);
77+
context.write(remainer_key, new IntWritable((int)remainer));
78+
}
79+
}
80+
}
81+
}

‎CRTMatrixA.java

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
import java.util.*;
2+
import java.lang.*;
3+
import java.io.*;
4+
5+
public class CRTMatrixA {
6+
int t,n,l,cnt,count,prefer;
7+
int[] data;
8+
int[] prime;
9+
List<List<Integer>> groups = new ArrayList<List<Integer>>(n);
10+
List<List<List<Integer>>> answers = new ArrayList<List<List<Integer>>>();
11+
public CRTMatrixA(int l, int n, int t,int[] prime){
12+
this.l = l;
13+
this.n = n;
14+
this.t = t;
15+
this.cnt = (t+1)*l;
16+
this.data = new int[cnt];
17+
for(int i=0;i<cnt;i++)
18+
this.data[i] = (i+1);
19+
this.count = 0;
20+
this.prime = prime;
21+
this.prefer = (int)((t+1)*l/n);
22+
}
23+
public int[][] run(){
24+
init();
25+
dfs(0);
26+
return findAns();
27+
}
28+
void init(){
29+
for (int i = 0; i < n; i++) {
30+
List<Integer> group = new ArrayList<Integer>();
31+
groups.add(group);
32+
}
33+
}
34+
void dfs(int pos){
35+
if(pos > cnt-1){
36+
for(int i=0;i<n;i++)
37+
if(0 == groups.get(i).size())
38+
return;
39+
boolean flag = true;
40+
for(int i=0;i<n;i++){
41+
Set<Integer> numbers = new HashSet<>();
42+
Iterator itr = groups.get(i).iterator();
43+
while(itr.hasNext()){
44+
int tmpnum = ((Integer)itr.next()).intValue()%l;
45+
Integer num = new Integer(tmpnum);
46+
if(numbers.contains(num))
47+
flag = false;
48+
else
49+
numbers.add(num);
50+
}
51+
}
52+
if(flag){
53+
List<List<Integer>> tmpgroups = new ArrayList<List<Integer>>(n);
54+
for(int i=0;i<n;i++){
55+
List<Integer> tmpgroup = new ArrayList<Integer>();
56+
if(groups.get(i).size()>prefer+1 || groups.get(i).size()<prefer)
57+
return;
58+
for(int j=0;j<groups.get(i).size();j++)
59+
tmpgroup.add(groups.get(i).get(j));
60+
tmpgroups.add(tmpgroup);
61+
}
62+
answers.add(tmpgroups);
63+
count++;
64+
}
65+
return;
66+
}
67+
groups.get(0).add(data[pos]);
68+
dfs(pos+1);
69+
for(int i=1;i<n;i++){
70+
if(groups.get(i-1).size()>1){
71+
groups.get(i-1).remove(groups.get(i-1).size()-1);
72+
groups.get(i).add(data[pos]);
73+
dfs(pos+1);
74+
if(i==n-1)
75+
groups.get(i).remove(groups.get(i).size()-1);
76+
}
77+
else{
78+
groups.get(i-1).remove(groups.get(i-1).size()-1);
79+
break;
80+
}
81+
}
82+
}
83+
int[][] findAns(){
84+
// System.out.println("CNT IS :" + count);
85+
int mxcnt = l;
86+
int ans = 0;
87+
int tmpmxcnt;
88+
int mxnum = 1;
89+
int tmpmxnum;
90+
for(int i=0;i<l;i++)
91+
mxnum *= prime[l-1];
92+
for(int i=0;i<count;i++){
93+
tmpmxcnt = 0;
94+
tmpmxnum = 0;
95+
// System.out.println("------------");
96+
for(int j=0;j<n;j++){
97+
int temp = 1;
98+
tmpmxcnt = Math.max(tmpmxcnt,answers.get(i).get(j).size());
99+
for(int k=0;k<answers.get(i).get(j).size();k++){
100+
// System.out.print(answers.get(i).get(j).get(k)+" ");
101+
temp *= prime[(answers.get(i).get(j).get(k)-1)%l];
102+
}
103+
// System.out.println();
104+
tmpmxnum = Math.max(tmpmxnum,temp);
105+
}
106+
if(mxcnt>=tmpmxcnt && mxnum>=tmpmxnum){
107+
mxcnt = tmpmxcnt;
108+
mxnum = tmpmxnum;
109+
ans = i;
110+
}
111+
}
112+
int[][] matrixA = new int[n][l];
113+
for(int i=0;i<n;i++){
114+
for(int j=0;j<answers.get(ans).get(i).size();j++){
115+
int tmp = (((Integer)answers.get(ans).get(i).get(j)).intValue()-1)%l;
116+
matrixA[i][tmp] = 1;
117+
}
118+
}
119+
return matrixA;
120+
}
121+
}

0 commit comments

Comments
 (0)