From b9d6ab9d1cc3017e45da48262bc135c5fd47b80b Mon Sep 17 00:00:00 2001 From: Lior Koskas Date: Thu, 22 Feb 2018 17:18:54 +0200 Subject: [PATCH] Add global configuration file --- GlobalConfigurations/conf.json | 10 +++ InstancesManagement/deploy_instances.py | 13 +++- .../Config_BMR.json | 0 .../Config_GMW.json | 0 .../Config_HighThroughputRing.json | 0 .../Config_LowCostConstantRoundMPC.json | 0 ...nfig_LowCostConstantRoundShortKeysMPC.json | 0 .../Config_MPCHonestMajority.json | 0 .../Config_MPCHonestMajorityNoTriples.json | 0 .../Config_MaliciousYaoBatch_OfflineP1.json | 0 .../Config_MaliciousYaoBatch_OfflineP2.json | 0 .../Config_MaliciousYaoBatch_OnlineP1.json | 0 .../Config_MaliciousYaoBatch_OnlineP2.json | 0 .../Config_PSI_items.json | 0 ...icatedSecretSharing3PartiesArithmetic.json | 0 ...retSharing3PartiesArithmeticNoTriples.json | 0 .../Config_SPDZ.json | 0 .../Config_SecretSharing.json | 0 .../Config_SemiHonestYao.json | 0 .../Config_YaoSingleExecution.json | 0 README.md | 77 +++++++++++-------- 21 files changed, 64 insertions(+), 36 deletions(-) create mode 100644 GlobalConfigurations/conf.json rename {Configurations => ProtocolsConfigurations}/Config_BMR.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_GMW.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_HighThroughputRing.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_LowCostConstantRoundMPC.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_LowCostConstantRoundShortKeysMPC.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_MPCHonestMajority.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_MPCHonestMajorityNoTriples.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_MaliciousYaoBatch_OfflineP1.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_MaliciousYaoBatch_OfflineP2.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_MaliciousYaoBatch_OnlineP1.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_MaliciousYaoBatch_OnlineP2.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_PSI_items.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_ReplicatedSecretSharing3PartiesArithmetic.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_ReplicatedSecretSharing3PartiesArithmeticNoTriples.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_SPDZ.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_SecretSharing.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_SemiHonestYao.json (100%) rename {Configurations => ProtocolsConfigurations}/Config_YaoSingleExecution.json (100%) diff --git a/GlobalConfigurations/conf.json b/GlobalConfigurations/conf.json new file mode 100644 index 00000000..88b7b9d1 --- /dev/null +++ b/GlobalConfigurations/conf.json @@ -0,0 +1,10 @@ +{ + "keys": + { + "key_0":"Matrixuseast1.pem" + }, + "securityGroups": + { + "securityGroup_0":"MatrixSGusest1" + } +} \ No newline at end of file diff --git a/InstancesManagement/deploy_instances.py b/InstancesManagement/deploy_instances.py index d1efa3c6..5f773d7a 100644 --- a/InstancesManagement/deploy_instances.py +++ b/InstancesManagement/deploy_instances.py @@ -78,6 +78,11 @@ def deploy_instances(): number_duplicated_servers = 0 spot_request = data['isSpotRequest'] + with open('GlobalConfigurations/conf.json') as gc_file: + global_config = json.load(gc_file, object_pairs_hook=OrderedDict) + keys = list(global_config['keys']) + security_group = list(global_config['securityGroup']) + if len(regions) > 1: number_of_instances = max(number_of_parties) // len(regions) if max(number_of_parties) % len(regions): @@ -117,8 +122,8 @@ def deploy_instances(): LaunchSpecification= { 'ImageId': amis_id[idx], - 'KeyName': 'Matrix%s' % regions[idx].replace('-', '')[:-1], - 'SecurityGroups': ['MatrixSG%s' % regions[idx].replace('-', '')[:-1]], + 'KeyName': keys[idx], + 'SecurityGroups': security_group[idx], 'InstanceType': machine_type, 'Placement': { @@ -131,10 +136,10 @@ def deploy_instances(): else: client.run_instances( ImageId=amis_id[idx], - KeyName='Matrix%s' % regions[idx].replace('-', '')[:-1], + KeyName=keys[idx], MinCount=int(number_of_instances_to_deploy), MaxCount=int(number_of_instances_to_deploy), - SecurityGroups=['MatrixSG%s' % regions[idx].replace('-', '')[:-1]], + SecurityGroups=security_group[idx], InstanceType=machine_type, Placement= { diff --git a/Configurations/Config_BMR.json b/ProtocolsConfigurations/Config_BMR.json similarity index 100% rename from Configurations/Config_BMR.json rename to ProtocolsConfigurations/Config_BMR.json diff --git a/Configurations/Config_GMW.json b/ProtocolsConfigurations/Config_GMW.json similarity index 100% rename from Configurations/Config_GMW.json rename to ProtocolsConfigurations/Config_GMW.json diff --git a/Configurations/Config_HighThroughputRing.json b/ProtocolsConfigurations/Config_HighThroughputRing.json similarity index 100% rename from Configurations/Config_HighThroughputRing.json rename to ProtocolsConfigurations/Config_HighThroughputRing.json diff --git a/Configurations/Config_LowCostConstantRoundMPC.json b/ProtocolsConfigurations/Config_LowCostConstantRoundMPC.json similarity index 100% rename from Configurations/Config_LowCostConstantRoundMPC.json rename to ProtocolsConfigurations/Config_LowCostConstantRoundMPC.json diff --git a/Configurations/Config_LowCostConstantRoundShortKeysMPC.json b/ProtocolsConfigurations/Config_LowCostConstantRoundShortKeysMPC.json similarity index 100% rename from Configurations/Config_LowCostConstantRoundShortKeysMPC.json rename to ProtocolsConfigurations/Config_LowCostConstantRoundShortKeysMPC.json diff --git a/Configurations/Config_MPCHonestMajority.json b/ProtocolsConfigurations/Config_MPCHonestMajority.json similarity index 100% rename from Configurations/Config_MPCHonestMajority.json rename to ProtocolsConfigurations/Config_MPCHonestMajority.json diff --git a/Configurations/Config_MPCHonestMajorityNoTriples.json b/ProtocolsConfigurations/Config_MPCHonestMajorityNoTriples.json similarity index 100% rename from Configurations/Config_MPCHonestMajorityNoTriples.json rename to ProtocolsConfigurations/Config_MPCHonestMajorityNoTriples.json diff --git a/Configurations/Config_MaliciousYaoBatch_OfflineP1.json b/ProtocolsConfigurations/Config_MaliciousYaoBatch_OfflineP1.json similarity index 100% rename from Configurations/Config_MaliciousYaoBatch_OfflineP1.json rename to ProtocolsConfigurations/Config_MaliciousYaoBatch_OfflineP1.json diff --git a/Configurations/Config_MaliciousYaoBatch_OfflineP2.json b/ProtocolsConfigurations/Config_MaliciousYaoBatch_OfflineP2.json similarity index 100% rename from Configurations/Config_MaliciousYaoBatch_OfflineP2.json rename to ProtocolsConfigurations/Config_MaliciousYaoBatch_OfflineP2.json diff --git a/Configurations/Config_MaliciousYaoBatch_OnlineP1.json b/ProtocolsConfigurations/Config_MaliciousYaoBatch_OnlineP1.json similarity index 100% rename from Configurations/Config_MaliciousYaoBatch_OnlineP1.json rename to ProtocolsConfigurations/Config_MaliciousYaoBatch_OnlineP1.json diff --git a/Configurations/Config_MaliciousYaoBatch_OnlineP2.json b/ProtocolsConfigurations/Config_MaliciousYaoBatch_OnlineP2.json similarity index 100% rename from Configurations/Config_MaliciousYaoBatch_OnlineP2.json rename to ProtocolsConfigurations/Config_MaliciousYaoBatch_OnlineP2.json diff --git a/Configurations/Config_PSI_items.json b/ProtocolsConfigurations/Config_PSI_items.json similarity index 100% rename from Configurations/Config_PSI_items.json rename to ProtocolsConfigurations/Config_PSI_items.json diff --git a/Configurations/Config_ReplicatedSecretSharing3PartiesArithmetic.json b/ProtocolsConfigurations/Config_ReplicatedSecretSharing3PartiesArithmetic.json similarity index 100% rename from Configurations/Config_ReplicatedSecretSharing3PartiesArithmetic.json rename to ProtocolsConfigurations/Config_ReplicatedSecretSharing3PartiesArithmetic.json diff --git a/Configurations/Config_ReplicatedSecretSharing3PartiesArithmeticNoTriples.json b/ProtocolsConfigurations/Config_ReplicatedSecretSharing3PartiesArithmeticNoTriples.json similarity index 100% rename from Configurations/Config_ReplicatedSecretSharing3PartiesArithmeticNoTriples.json rename to ProtocolsConfigurations/Config_ReplicatedSecretSharing3PartiesArithmeticNoTriples.json diff --git a/Configurations/Config_SPDZ.json b/ProtocolsConfigurations/Config_SPDZ.json similarity index 100% rename from Configurations/Config_SPDZ.json rename to ProtocolsConfigurations/Config_SPDZ.json diff --git a/Configurations/Config_SecretSharing.json b/ProtocolsConfigurations/Config_SecretSharing.json similarity index 100% rename from Configurations/Config_SecretSharing.json rename to ProtocolsConfigurations/Config_SecretSharing.json diff --git a/Configurations/Config_SemiHonestYao.json b/ProtocolsConfigurations/Config_SemiHonestYao.json similarity index 100% rename from Configurations/Config_SemiHonestYao.json rename to ProtocolsConfigurations/Config_SemiHonestYao.json diff --git a/Configurations/Config_YaoSingleExecution.json b/ProtocolsConfigurations/Config_YaoSingleExecution.json similarity index 100% rename from Configurations/Config_YaoSingleExecution.json rename to ProtocolsConfigurations/Config_YaoSingleExecution.json diff --git a/README.md b/README.md index 3723a8e1..b382488e 100644 --- a/README.md +++ b/README.md @@ -37,42 +37,17 @@ To install under CentOS 7.3: After You installed python 3 and pip3 you will need to install the modules MATRIX uses. To install this three modules use pip3 -`pip3 install --user openpyxl fabric fabric3` +`pip3 install --user openpyxl fabric fabric3 boto3 colorama` + +**NOTE**: on some computers the following error may appear: `locale.Error: unsupported locale setting` +To fix it, run: +1. sudo apt-get clean && sudo apt-get update && sudo apt-get install -y locales +2. locale-gen en_US.UTF-8 After the modules installed, clone this repository to install MATRIX on your system. ## MATRIX Modules -### Configurations -MATRIX uses configuration file to set it execution. The configuration file is written in [json](https://en.wikipedia.org/wiki/JSON) format. -Each configuration file has the following fields: -* amis - List of AMI(s). -If using custom AWS AMI, the AMI need to be installed in all the regions that we want to test. -* protocol - Name of protocol -* numOfParties - Size of the parties -* gitAddress - Git repository path. MATRIX will clone the repository into all target servers, configure, -make and install. If installation of other libraries is needed to be done, see pre-process section of MATRIX for details. -* awsInstType - AWS instance type. For details about the different instance types, you can read [here](https://aws.amazon.com/ec2/instance-types/). -* awsBidPrice - Bid price for spot instances machine in USD -* executableName - The name of the executable to execute -* preProcessTask - ID of the pre process task that required. -The available pre process tasks that defines in MATRIX can be found in this [script](../master/ExperimentExecute/pre_process.py) -* Configurations - List of configurations to run. Each configuration is a set of CLI arguments to the executable. -The arguments are separated between them by '@'. Party ID is added automatically -* numOfRepetitions - How Many times MATRIX will execute the protocol -* numOfInternalRepetitions - How many times the protocol will be executed on single run. -* gitBranch - The branch the protocol uses. default value is `master` -* isPublished - Indicate if the protocol was published. -* isExternal - Indicate if the protocol external to libscapi library -* regions - AWS regions to execute the protocol. -* workingDirectory - The directory of the protocol and the data related to the protocol. -* resultsDirectory - Directory to copy to the results files from the servers. The directory is local directory at the MATRIX system computer. -* emails - MATRIX will send notifications to this email addresses. Multiple email addresses are supported -* institute - Research Group identifier -* coordinatorConfig - If coordinator exists in the protocol, the configuration for him will described here. -The configuration need to be in the same format of 'configurations' field -* coordinatorExecutable - The name of the coordinator executable - ### InstancesManagement After config file was created, You will need to deploy your images(instances). MATRIX supports three different deployments: @@ -82,7 +57,7 @@ After config file was created, You will need to deploy your images(instances). M #### Local Deployment -To deploy MATRIX locally set `regions` parameter to `local` at the config [file](../master/Configurations/Config_BMR.json) +To deploy MATRIX locally set `regions` parameter to `local` at the config [file](../master/ProtocolsConfigurations/Config_BMR.json) #### AWS Deployment @@ -127,6 +102,44 @@ The values for analysedParameter are: Example file can be found [here](../master/Assets/MPCHonestMajorityNoTriples_cpu_partyId=0_numOfParties=3.json) +## MATRIX Configurations + +### GlobalConfigurations + +In order to connect to the instances MATRIX uses a file that contains the AWS keys and security groups. +For each region in AWS you need to create an entry in the global configuration file. +Sample configuration file can be found [here](../master/GlobalConfigurations/conf.json) + +### ProtocolsConfigurations +MATRIX uses configuration file to set it execution. The configuration file is written in [json](https://en.wikipedia.org/wiki/JSON) format. +Each configuration file has the following fields: +* amis - List of AMI(s). +If using custom AWS AMI, the AMI need to be installed in all the regions that we want to test. +* protocol - Name of protocol +* numOfParties - Size of the parties +* gitAddress - Git repository path. MATRIX will clone the repository into all target servers, configure, +make and install. If installation of other libraries is needed to be done, see pre-process section of MATRIX for details. +* awsInstType - AWS instance type. For details about the different instance types, you can read [here](https://aws.amazon.com/ec2/instance-types/). +* awsBidPrice - Bid price for spot instances machine in USD +* executableName - The name of the executable to execute +* preProcessTask - ID of the pre process task that required. +The available pre process tasks that defines in MATRIX can be found in this [script](../master/ExperimentExecute/pre_process.py) +* Configurations - List of configurations to run. Each configuration is a set of CLI arguments to the executable. +The arguments are separated between them by '@'. Party ID is added automatically +* numOfRepetitions - How Many times MATRIX will execute the protocol +* numOfInternalRepetitions - How many times the protocol will be executed on single run. +* gitBranch - The branch the protocol uses. default value is `master` +* isPublished - Indicate if the protocol was published. +* isExternal - Indicate if the protocol external to libscapi library +* regions - AWS regions to execute the protocol. +* workingDirectory - The directory of the protocol and the data related to the protocol. +* resultsDirectory - Directory to copy to the results files from the servers. The directory is local directory at the MATRIX system computer. +* emails - MATRIX will send notifications to this email addresses. Multiple email addresses are supported +* institute - Research Group identifier +* coordinatorConfig - If coordinator exists in the protocol, the configuration for him will described here. +The configuration need to be in the same format of 'configurations' field +* coordinatorExecutable - The name of the coordinator executable + ### MATRIX Usage