Better Server Memory Management & Programs can enque precompiled code

This commit is contained in:
mrbesen 2018-04-25 18:23:03 +02:00
parent b8e779927c
commit 3aeba6c8f2
9 changed files with 156 additions and 57 deletions

File diff suppressed because one or more lines are too long

Binary file not shown.

View File

@ -67,7 +67,7 @@ public class Client implements Runnable{
soc.close(); soc.close();
System.out.println("Disconnected!"); System.out.println("Disconnected!");
//re-enque all take jobs //re-enque all take jobs
Server.getServer().getProgram().jobmanager.reenque(takenjobs); Server.getServer().getProgram().getJobManager().reenque(takenjobs);
takenjobs.clear(); takenjobs.clear();
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
@ -76,13 +76,14 @@ public class Client implements Runnable{
@Override @Override
public void run() { public void run() {
while(!soc.isClosed() & hold_connection) { while(!soc.isClosed() && hold_connection) {
//handle data //handle data
try { try {
handler.HandleData((Data) in.readObject(), this); handler.HandleData((Data) in.readObject(), this);
// System.out.println("Recived data!"); // System.out.println("Recived data!");
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); // e.printStackTrace();
System.err.println("Error while reciving! Disconnecting Client: " + soc.getInetAddress() );
disconnect(); disconnect();
} }
} }

View File

@ -46,7 +46,7 @@ public class Server implements PacketHandler {
connections.add(c); connections.add(c);
execution.submit(c); execution.submit(c);
c.send(new Data("welcomme")); c.send(new Data("welcomme"));
prog.jobmanager.update(); prog.getJobManager().update();
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
@ -60,8 +60,8 @@ public class Server implements PacketHandler {
programthread = new Thread(new Runnable() { programthread = new Thread(new Runnable() {
@Override @Override
public void run() { public void run() {
prog.run(); for(int i = 0; i < 15 && prog.enquenextJob(); i++);//enque 15 start jobs
System.out.println("Program enqued " + prog.jobmanager.jobs_total() + " Jobs."); System.out.println("Program enqued " + prog.getJobManager().jobs_total() + " Jobs.");
} }
}, "Programm Thread"); }, "Programm Thread");
programthread.start(); programthread.start();
@ -114,8 +114,8 @@ public class Server implements PacketHandler {
c.send(new Data("pingback")); c.send(new Data("pingback"));
// System.out.println("Pinged, pining back..."); // System.out.println("Pinged, pining back...");
} else if(info.equalsIgnoreCase("nextplease")) { } else if(info.equalsIgnoreCase("nextplease")) {
if(prog.jobmanager.hasNext()) { if(prog.getJobManager().hasNext()) {
c.send(new Data(prog.jobmanager.next(),ContentType.Job)); c.send(new Data(prog.getJobManager().next(),ContentType.Job));
} else { } else {
System.out.println("Out of Tasks"); System.out.println("Out of Tasks");
} }

View File

@ -1,12 +1,26 @@
package Core; package Core;
import java.nio.ByteBuffer;
import java.util.Arrays;
import Job.JobManager; import Job.JobManager;
import Job.Result; import Job.Result;
import Job.Result.PartialResult; import Job.Result.PartialResult;
public abstract class Program implements Runnable { public abstract class Program{
public JobManager jobmanager = new JobManager(); private JobManager jobmanager = new JobManager(this);
public JobManager getJobManager() {
return jobmanager;
}
/**
* Called by the JobManager or Server, to get the Program generating the next Job.
* The Program is allowed to sumbit more than one Program to the JobManager.
* returns false, when all jobs are submited
*/
public abstract boolean enquenextJob();
public void HandleResult(Result r) { public void HandleResult(Result r) {
jobmanager.setdone(r.getJobId()); jobmanager.setdone(r.getJobId());
@ -20,5 +34,26 @@ public abstract class Program implements Runnable {
} }
} }
protected byte[] inttoBytes(int i ) {
return ByteBuffer.allocate(4).putInt(i).array();
}
protected byte[] longtoBytes(long i ) {
return ByteBuffer.allocate(8).putLong(i).array();
}
/**
* removes the zero terminating byte
* @param s
* @return
*/
protected byte[] stringtoBytes(String s) {
byte[] tmp = s.getBytes();
if(tmp[tmp.length-1] == 0) {
tmp = Arrays.copyOf(tmp, tmp.length-1);
}
return tmp;
}
public abstract void HandlePartialResult(PartialResult pres); public abstract void HandlePartialResult(PartialResult pres);
} }

View File

@ -53,7 +53,7 @@ public class Starter {
Server.getServer().close(); Server.getServer().close();
run = false; run = false;
} else if(in.equalsIgnoreCase("stats")) { } else if(in.equalsIgnoreCase("stats")) {
JobManager jm = server.getProgram().jobmanager; JobManager jm = server.getProgram().getJobManager();
System.out.println("\nStats:\nTasks done : " + jm.jobs_done() + "\nTasks send : " + jm.jobs_send() + "\nTasks compiled: " + jm.jobs_compiled() + "/" + jm.jobs_compiledtarget() + "\nCurrently Compiling: " + jm.isCompiling() + "\nTasks enqued : " + jm.jobs_enqued() + "\n--------------------\ntotal : " + jm.jobs_total()+ "\nConnections: " + server.getConnectionCount()); System.out.println("\nStats:\nTasks done : " + jm.jobs_done() + "\nTasks send : " + jm.jobs_send() + "\nTasks compiled: " + jm.jobs_compiled() + "/" + jm.jobs_compiledtarget() + "\nCurrently Compiling: " + jm.isCompiling() + "\nTasks enqued : " + jm.jobs_enqued() + "\n--------------------\ntotal : " + jm.jobs_total()+ "\nConnections: " + server.getConnectionCount());
} else { } else {
System.out.println("unknown Command."); System.out.println("unknown Command.");

View File

@ -10,6 +10,7 @@ import java.util.List;
import java.util.Scanner; import java.util.Scanner;
import Comunication.Server; import Comunication.Server;
import Core.Program;
import Utils.StringUtils; import Utils.StringUtils;
public class JobManager implements Iterator<Job>{ public class JobManager implements Iterator<Job>{
@ -19,20 +20,28 @@ public class JobManager implements Iterator<Job>{
private List<Job> send = new ArrayList<Job>();//assigned jobs private List<Job> send = new ArrayList<Job>();//assigned jobs
private List<Job> done = new ArrayList<Job>();//done jobs private List<Job> done = new ArrayList<Job>();//done jobs
private Program prog;
private int jobcount = 0; private int jobcount = 0;
private boolean isCompiling = false; private boolean isCompiling = false;
public JobManager(Program program) {
prog = program;
}
public void reenque(List<Job> jobs) { public void reenque(List<Job> jobs) {
todo.addAll(0, jobs); todo.addAll(0, jobs);
send.removeAll(jobs); send.removeAll(jobs);
update(); update();
} }
public void enque(Job newjob) { public void enque(Job newjob, boolean isPreCompiled) {
jobcount++; jobcount++;
newjob.setId(jobcount); newjob.setId(jobcount);
enqued.add(newjob); if(isPreCompiled)
todo.add(newjob);
else
enqued.add(newjob);
update(); update();
} }
@ -45,9 +54,13 @@ public class JobManager implements Iterator<Job>{
@Override @Override
public void run() { public void run() {
isCompiling = true;//doppelt hält besser isCompiling = true;//doppelt hält besser
// System.out.println("Compilingque started!"); //compile them!
while(enqued.size() > 0 & todo.size() < 15 & isCompiling) { while(enqued.size() > 0 && todo.size() < jobs_compiledtarget() && isCompiling) {
Job j = enqued.get(0); Job j = enqued.get(0);
if(j == null) {
System.out.println("enqued.size(): " + enqued.size() + " but job.get(0) is null!");
break;
}
//compile script //compile script
try { try {
// System.out.println(j.code); // System.out.println(j.code);
@ -86,20 +99,17 @@ public class JobManager implements Iterator<Job>{
enqued.remove(0);//manage ques enqued.remove(0);//manage ques
todo.add(j); todo.add(j);
//br.close();
if(classfile.exists())//delete all files if(classfile.exists())//delete all files
classfile.delete(); classfile.delete();
if (file.exists()) if (file.exists())
file.delete(); file.delete();
// System.out.println("Compile done.");
} catch(IOException | InterruptedException e) { } catch(IOException | InterruptedException e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
isCompiling = false; isCompiling = false;
// System.out.println("Compiling que Stopped!"); // System.out.println("Compiler stopped!");
} }
}, "Compiler"); }, "Compiler");
} }
@ -109,20 +119,36 @@ public class JobManager implements Iterator<Job>{
*/ */
@Override @Override
public boolean hasNext() { public boolean hasNext() {
return todo.size() != 0; if(todo.size() == 0) {
update();
return false;
}
return true;
} }
@Override @Override
public Job next() { public Job next() {
//Warning! not checked if todo.get(0) exists!
send.add(todo.get(0)); send.add(todo.get(0));
todo.remove(0); todo.remove(0);
update(); update();
System.out.println("" + ((int) (((float) done.size()) / ((float)jobs_total()))*100) + "% Done"); System.out.println(((int) (((float) done.size()) / ((float)jobs_total()))*100) + "% Done");
return send.get(send.size()-1); return send.get(send.size()-1);
} }
public void update() {//called from Server on new Client Connection /**
if(enqued.size() > 0 & !isCompiling & todo.size() < jobs_compiledtarget()) {//7 für jede connection vorrätig * Called, when the job-lists get modified or a new client connects.
* This method simply checks weather the compiler-thread should be started or not.
* If the Thread should be started, this method does it with no further instruction.
*/
public void update() {
boolean hasnext = true;//variable zum testen, ob das Programm weitere Jobs hat
while(enqued.size() + todo.size() < jobs_compiledtarget() && (hasnext = prog.enquenextJob()));//genarte new jobs!
if(!hasnext) {
System.out.println("Program has no new Jobs!");
//TODO: somehow stop the compile thread after compiling (and disable it from re-opening) and set a flag, that after these jobs are done the server could shutdown.
}
if(enqued.size() > 0 && !isCompiling && todo.size() < jobs_compiledtarget()) {//7 jobs for each connection and at least 10
startCompile(); startCompile();
} }
} }
@ -185,7 +211,7 @@ public class JobManager implements Iterator<Job>{
* @param jobId * @param jobId
*/ */
public void setdone(int jobId) { public void setdone(int jobId) {
//find yob //find job
Job j = null; Job j = null;
int pos = -1; int pos = -1;
for(int i = 0; i < send.size(); i++) { for(int i = 0; i < send.size(); i++) {
@ -199,6 +225,7 @@ public class JobManager implements Iterator<Job>{
if(j != null) { if(j != null) {
send.remove(pos); send.remove(pos);
done.add(j); done.add(j);
//TODO: delete old done jobs, to preverse Memory?
} else { } else {
System.out.println("Job id " + jobId + " not found"); System.out.println("Job id " + jobId + " not found");
} }

View File

@ -4,12 +4,14 @@ import Job.Result.PartialResult;
public class Test extends Program { public class Test extends Program {
private int c = 0;
@Override @Override
public void run() { public boolean enquenextJob() {
for(int c = 5; c < 1000; c++) { String code = "import Job.Job;\nimport Job.Result;\nimport Job.Result.PartialResult;\nimport Job.Jobsrc;\npublic class A" + c + " extends Jobsrc {\n@Override\npublic Result run() {\ndouble i ="+c+";\nfor(int n = 2; n < " + c + ";n++){\n i *= n;\n}\n Result r = new Result(" + (getJobManager().jobs_total() +1) +");\nr.OutputConsole(\"!"+c+"= \"+i);\n return r;\n }}";
String code = "import Job.Job;\nimport Job.Result;\nimport Job.Result.PartialResult;\nimport Job.Jobsrc;\npublic class A" + c + " extends Jobsrc {\n@Override\npublic Result run() {\ndouble i ="+c+";\nfor(int n = 2; n < " + c + ";n++){\n i *= n;\n}\n Result r = new Result(" + (jobmanager.jobs_total() +1) +");\nr.OutputConsole(\"!"+c+"= \"+i);\n return r;\n }}"; getJobManager().enque(new Job(code), false);
jobmanager.enque(new Job(code)); c++;
} return true;
} }
@Override @Override

View File

@ -15,14 +15,13 @@ import Job.Result.PartialResult;
public class PrimeFinder extends Program { public class PrimeFinder extends Program {
List<Double> primes = new ArrayList<Double>(); List<Double> primes = new ArrayList<Double>();
double count = -1; double count = -1;//current prime
File file = new File("primes"); File file = new File("primes");
int save_count;//to save every 20 times int save_count;//to save every 20 times
int save_intervall = 100; int save_intervall = 100;
@Override public PrimeFinder() {
public void run() {
//load primes //load primes
try { try {
if(file.exists()) { if(file.exists()) {
@ -51,7 +50,7 @@ public class PrimeFinder extends Program {
count = primes.get(primes.size()-1)+1; count = primes.get(primes.size()-1)+1;
String insert = PrimetoString(count); String insert = PrimetoString(count);
jobmanager.enque(new Job("import Job.Jobsrc;\nimport Job.Result;\nimport Job.Result.ResultType;\npublic class A" + insert + " extends Jobsrc{\n double num = " + insert + ";\n @Override\n public Result run() {\n Result r = new Result();\n for(double test = num; test < num + 200; test = test +2) {\n for(double i = 2; i * i <= test+2; i++) {\n if(test%i == 0)\n r.add(r.new PartialResult(ResultType.Value, test + \"|\" + false));\n }\n r.add(r.new PartialResult(ResultType.Value, test + \"|\" + true));\n }\n return r;\n }\n}") ); getJobManager().enque(new Job("import Job.Jobsrc;\nimport Job.Result;\nimport Job.Result.ResultType;\npublic class A" + insert + " extends Jobsrc{\n double num = " + insert + ";\n @Override\n public Result run() {\n Result r = new Result();\n for(double test = num; test < num + 200; test = test +2) {\n for(double i = 2; i * i <= test+2; i++) {\n if(test%i == 0)\n r.add(r.new PartialResult(ResultType.Value, test + \"|\" + false));\n }\n r.add(r.new PartialResult(ResultType.Value, test + \"|\" + true));\n }\n return r;\n }\n}"), false);
count+= 200; count+= 200;
} }
@ -67,11 +66,9 @@ public class PrimeFinder extends Program {
fw.close(); fw.close();
System.out.println("Primes saved"); System.out.println("Primes saved");
} }
private String PrimetoString(double prime) { private String PrimetoString(double prime) {
String out = prime + ""; return ((int) prime) / 100 + "";
out = out.substring(0,out.length()-2);
return out;
} }
@Override @Override
@ -87,7 +84,7 @@ public class PrimeFinder extends Program {
primes.add(Double.parseDouble(split[0])); primes.add(Double.parseDouble(split[0]));
} }
} }
save_count++; save_count++;
if(save_count >=save_intervall) if(save_count >=save_intervall)
try { try {
@ -96,4 +93,11 @@ public class PrimeFinder extends Program {
e.printStackTrace(); e.printStackTrace();
} }
} }
@Override
public boolean enquenextJob() {
checkforprime();
return true;
}
} }