diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..c9bec57 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include mxeval/data *.json *.jsonl + diff --git a/README.md b/README.md index 1d8e092..a4d1a71 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,7 @@ Below we demonstrate the language conversion (component A above) for the convers Check out and install this repository: ``` -git clone https://github.com/amazon-science/mxeval.git -pip install -e mxeval +pip install git+https://github.com/amazon-science/mxeval.git ``` diff --git a/mxeval/data.py b/mxeval/data.py index 7b4122b..3cde822 100644 --- a/mxeval/data.py +++ b/mxeval/data.py @@ -5,10 +5,10 @@ ROOT = os.path.dirname(os.path.abspath(__file__)) -MULTILINGUAL_HUMANEVAL_METADATA = os.path.join(ROOT, "..", "data", "multilingual_humaneval", "metadata.json") +MULTILINGUAL_HUMANEVAL_METADATA = os.path.join(ROOT, "data", "multilingual_humaneval", "metadata.json") with open(MULTILINGUAL_HUMANEVAL_METADATA, "r", encoding="utf-8") as fr: MULTILINGUAL_HUMANEVAL_METADATA = json.load(fr) -HUMAN_EVAL_PYTHON = os.path.join(ROOT, "..", "data", "multilingual_humaneval", MULTILINGUAL_HUMANEVAL_METADATA["python"]) +HUMAN_EVAL_PYTHON = os.path.join(ROOT, "data", "multilingual_humaneval", MULTILINGUAL_HUMANEVAL_METADATA["python"]) HUMAN_EVAL = HUMAN_EVAL_PYTHON @@ -61,7 +61,7 @@ def get_metadata(dataset, metadata_type="problem"): "mathqa-x": "multilingual_mathqa"} typemap = {"problem": "metadata.json", "example": "metadata_examples.json"} - datadir = os.path.join(ROOT, "..", "data", dataset_dirmap[dataset]) + datadir = os.path.join(ROOT, "data", dataset_dirmap[dataset]) path = os.path.join(datadir, typemap[metadata_type]) with open(path, "r") as f: metadata = json.load(f) diff --git a/data/mbxp/LICENSE b/mxeval/data/mbxp/LICENSE similarity index 100% rename from data/mbxp/LICENSE rename to mxeval/data/mbxp/LICENSE diff --git a/data/mbxp/examples/mbcpp_samples.jsonl b/mxeval/data/mbxp/examples/mbcpp_samples.jsonl similarity index 100% rename from data/mbxp/examples/mbcpp_samples.jsonl rename to mxeval/data/mbxp/examples/mbcpp_samples.jsonl diff --git a/data/mbxp/examples/mbcsp_samples.jsonl b/mxeval/data/mbxp/examples/mbcsp_samples.jsonl similarity index 100% rename from data/mbxp/examples/mbcsp_samples.jsonl rename to mxeval/data/mbxp/examples/mbcsp_samples.jsonl diff --git a/data/mbxp/examples/mbgp_samples.jsonl b/mxeval/data/mbxp/examples/mbgp_samples.jsonl similarity index 100% rename from data/mbxp/examples/mbgp_samples.jsonl rename to mxeval/data/mbxp/examples/mbgp_samples.jsonl diff --git a/data/mbxp/examples/mbjp_samples.jsonl b/mxeval/data/mbxp/examples/mbjp_samples.jsonl similarity index 100% rename from data/mbxp/examples/mbjp_samples.jsonl rename to mxeval/data/mbxp/examples/mbjp_samples.jsonl diff --git a/data/mbxp/examples/mbjsp_samples.jsonl b/mxeval/data/mbxp/examples/mbjsp_samples.jsonl similarity index 100% rename from data/mbxp/examples/mbjsp_samples.jsonl rename to mxeval/data/mbxp/examples/mbjsp_samples.jsonl diff --git a/data/mbxp/examples/mbkp_samples.jsonl b/mxeval/data/mbxp/examples/mbkp_samples.jsonl similarity index 100% rename from data/mbxp/examples/mbkp_samples.jsonl rename to mxeval/data/mbxp/examples/mbkp_samples.jsonl diff --git a/data/mbxp/examples/mbphp_samples.jsonl b/mxeval/data/mbxp/examples/mbphp_samples.jsonl similarity index 100% rename from data/mbxp/examples/mbphp_samples.jsonl rename to mxeval/data/mbxp/examples/mbphp_samples.jsonl diff --git a/data/mbxp/examples/mbplp_samples.jsonl b/mxeval/data/mbxp/examples/mbplp_samples.jsonl similarity index 100% rename from data/mbxp/examples/mbplp_samples.jsonl rename to mxeval/data/mbxp/examples/mbplp_samples.jsonl diff --git a/data/mbxp/examples/mbpp_samples.jsonl b/mxeval/data/mbxp/examples/mbpp_samples.jsonl similarity index 100% rename from data/mbxp/examples/mbpp_samples.jsonl rename to mxeval/data/mbxp/examples/mbpp_samples.jsonl diff --git a/data/mbxp/examples/mbrbp_samples.jsonl b/mxeval/data/mbxp/examples/mbrbp_samples.jsonl similarity index 100% rename from data/mbxp/examples/mbrbp_samples.jsonl rename to mxeval/data/mbxp/examples/mbrbp_samples.jsonl diff --git a/data/mbxp/examples/mbscp_samples.jsonl b/mxeval/data/mbxp/examples/mbscp_samples.jsonl similarity index 100% rename from data/mbxp/examples/mbscp_samples.jsonl rename to mxeval/data/mbxp/examples/mbscp_samples.jsonl diff --git a/data/mbxp/examples/mbswp_samples.jsonl b/mxeval/data/mbxp/examples/mbswp_samples.jsonl similarity index 100% rename from data/mbxp/examples/mbswp_samples.jsonl rename to mxeval/data/mbxp/examples/mbswp_samples.jsonl diff --git a/data/mbxp/examples/mbtsp_samples.jsonl b/mxeval/data/mbxp/examples/mbtsp_samples.jsonl similarity index 100% rename from data/mbxp/examples/mbtsp_samples.jsonl rename to mxeval/data/mbxp/examples/mbtsp_samples.jsonl diff --git a/data/mbxp/fewshot_prompts/cpp_fewshot.cpp b/mxeval/data/mbxp/fewshot_prompts/cpp_fewshot.cpp similarity index 100% rename from data/mbxp/fewshot_prompts/cpp_fewshot.cpp rename to mxeval/data/mbxp/fewshot_prompts/cpp_fewshot.cpp diff --git a/data/mbxp/fewshot_prompts/csharp_fewshot.cs b/mxeval/data/mbxp/fewshot_prompts/csharp_fewshot.cs similarity index 100% rename from data/mbxp/fewshot_prompts/csharp_fewshot.cs rename to mxeval/data/mbxp/fewshot_prompts/csharp_fewshot.cs diff --git a/data/mbxp/fewshot_prompts/go_fewshot.go b/mxeval/data/mbxp/fewshot_prompts/go_fewshot.go similarity index 100% rename from data/mbxp/fewshot_prompts/go_fewshot.go rename to mxeval/data/mbxp/fewshot_prompts/go_fewshot.go diff --git a/data/mbxp/fewshot_prompts/java_fewshot_v1.java b/mxeval/data/mbxp/fewshot_prompts/java_fewshot_v1.java similarity index 100% rename from data/mbxp/fewshot_prompts/java_fewshot_v1.java rename to mxeval/data/mbxp/fewshot_prompts/java_fewshot_v1.java diff --git a/data/mbxp/fewshot_prompts/javascript_fewshot_v1.js b/mxeval/data/mbxp/fewshot_prompts/javascript_fewshot_v1.js similarity index 100% rename from data/mbxp/fewshot_prompts/javascript_fewshot_v1.js rename to mxeval/data/mbxp/fewshot_prompts/javascript_fewshot_v1.js diff --git a/data/mbxp/fewshot_prompts/kotlin_fewshot_v1.kt b/mxeval/data/mbxp/fewshot_prompts/kotlin_fewshot_v1.kt similarity index 100% rename from data/mbxp/fewshot_prompts/kotlin_fewshot_v1.kt rename to mxeval/data/mbxp/fewshot_prompts/kotlin_fewshot_v1.kt diff --git a/data/mbxp/fewshot_prompts/perl_fewshot.perl b/mxeval/data/mbxp/fewshot_prompts/perl_fewshot.perl similarity index 100% rename from data/mbxp/fewshot_prompts/perl_fewshot.perl rename to mxeval/data/mbxp/fewshot_prompts/perl_fewshot.perl diff --git a/data/mbxp/fewshot_prompts/php_fewshot_v1.php b/mxeval/data/mbxp/fewshot_prompts/php_fewshot_v1.php similarity index 100% rename from data/mbxp/fewshot_prompts/php_fewshot_v1.php rename to mxeval/data/mbxp/fewshot_prompts/php_fewshot_v1.php diff --git a/data/mbxp/fewshot_prompts/python_fewshot_v1.py b/mxeval/data/mbxp/fewshot_prompts/python_fewshot_v1.py similarity index 100% rename from data/mbxp/fewshot_prompts/python_fewshot_v1.py rename to mxeval/data/mbxp/fewshot_prompts/python_fewshot_v1.py diff --git a/data/mbxp/fewshot_prompts/ruby_fewshot_v1.rb b/mxeval/data/mbxp/fewshot_prompts/ruby_fewshot_v1.rb similarity index 100% rename from data/mbxp/fewshot_prompts/ruby_fewshot_v1.rb rename to mxeval/data/mbxp/fewshot_prompts/ruby_fewshot_v1.rb diff --git a/data/mbxp/fewshot_prompts/scala_fewshot.scala b/mxeval/data/mbxp/fewshot_prompts/scala_fewshot.scala similarity index 100% rename from data/mbxp/fewshot_prompts/scala_fewshot.scala rename to mxeval/data/mbxp/fewshot_prompts/scala_fewshot.scala diff --git a/data/mbxp/fewshot_prompts/swift_fewshot.swift b/mxeval/data/mbxp/fewshot_prompts/swift_fewshot.swift similarity index 100% rename from data/mbxp/fewshot_prompts/swift_fewshot.swift rename to mxeval/data/mbxp/fewshot_prompts/swift_fewshot.swift diff --git a/data/mbxp/fewshot_prompts/typescript_fewshot.ts b/mxeval/data/mbxp/fewshot_prompts/typescript_fewshot.ts similarity index 100% rename from data/mbxp/fewshot_prompts/typescript_fewshot.ts rename to mxeval/data/mbxp/fewshot_prompts/typescript_fewshot.ts diff --git a/data/mbxp/mbcpp_release_v1.2.jsonl b/mxeval/data/mbxp/mbcpp_release_v1.2.jsonl similarity index 100% rename from data/mbxp/mbcpp_release_v1.2.jsonl rename to mxeval/data/mbxp/mbcpp_release_v1.2.jsonl diff --git a/data/mbxp/mbcsp_release_v1.2.jsonl b/mxeval/data/mbxp/mbcsp_release_v1.2.jsonl similarity index 100% rename from data/mbxp/mbcsp_release_v1.2.jsonl rename to mxeval/data/mbxp/mbcsp_release_v1.2.jsonl diff --git a/data/mbxp/mbgp_release_v1.1.jsonl b/mxeval/data/mbxp/mbgp_release_v1.1.jsonl similarity index 100% rename from data/mbxp/mbgp_release_v1.1.jsonl rename to mxeval/data/mbxp/mbgp_release_v1.1.jsonl diff --git a/data/mbxp/mbjp_release_v1.2.jsonl b/mxeval/data/mbxp/mbjp_release_v1.2.jsonl similarity index 100% rename from data/mbxp/mbjp_release_v1.2.jsonl rename to mxeval/data/mbxp/mbjp_release_v1.2.jsonl diff --git a/data/mbxp/mbjsp_release_v1.2.jsonl b/mxeval/data/mbxp/mbjsp_release_v1.2.jsonl similarity index 100% rename from data/mbxp/mbjsp_release_v1.2.jsonl rename to mxeval/data/mbxp/mbjsp_release_v1.2.jsonl diff --git a/data/mbxp/mbkp_release_v1.2.jsonl b/mxeval/data/mbxp/mbkp_release_v1.2.jsonl similarity index 100% rename from data/mbxp/mbkp_release_v1.2.jsonl rename to mxeval/data/mbxp/mbkp_release_v1.2.jsonl diff --git a/data/mbxp/mbphp_release_v1.2.jsonl b/mxeval/data/mbxp/mbphp_release_v1.2.jsonl similarity index 100% rename from data/mbxp/mbphp_release_v1.2.jsonl rename to mxeval/data/mbxp/mbphp_release_v1.2.jsonl diff --git a/data/mbxp/mbplp_release_v1.jsonl b/mxeval/data/mbxp/mbplp_release_v1.jsonl similarity index 100% rename from data/mbxp/mbplp_release_v1.jsonl rename to mxeval/data/mbxp/mbplp_release_v1.jsonl diff --git a/data/mbxp/mbpp_release_v1.jsonl b/mxeval/data/mbxp/mbpp_release_v1.jsonl similarity index 100% rename from data/mbxp/mbpp_release_v1.jsonl rename to mxeval/data/mbxp/mbpp_release_v1.jsonl diff --git a/data/mbxp/mbrbp_release_v1.2.jsonl b/mxeval/data/mbxp/mbrbp_release_v1.2.jsonl similarity index 100% rename from data/mbxp/mbrbp_release_v1.2.jsonl rename to mxeval/data/mbxp/mbrbp_release_v1.2.jsonl diff --git a/data/mbxp/mbscp_release_v1.jsonl b/mxeval/data/mbxp/mbscp_release_v1.jsonl similarity index 100% rename from data/mbxp/mbscp_release_v1.jsonl rename to mxeval/data/mbxp/mbscp_release_v1.jsonl diff --git a/data/mbxp/mbswp_release_v1.jsonl b/mxeval/data/mbxp/mbswp_release_v1.jsonl similarity index 100% rename from data/mbxp/mbswp_release_v1.jsonl rename to mxeval/data/mbxp/mbswp_release_v1.jsonl diff --git a/data/mbxp/mbtsp_release_v1.2.jsonl b/mxeval/data/mbxp/mbtsp_release_v1.2.jsonl similarity index 100% rename from data/mbxp/mbtsp_release_v1.2.jsonl rename to mxeval/data/mbxp/mbtsp_release_v1.2.jsonl diff --git a/data/mbxp/metadata.json b/mxeval/data/mbxp/metadata.json similarity index 100% rename from data/mbxp/metadata.json rename to mxeval/data/mbxp/metadata.json diff --git a/data/mbxp/metadata_examples.json b/mxeval/data/mbxp/metadata_examples.json similarity index 100% rename from data/mbxp/metadata_examples.json rename to mxeval/data/mbxp/metadata_examples.json diff --git a/data/multilingual_humaneval/HumanEval.jsonl b/mxeval/data/multilingual_humaneval/HumanEval.jsonl similarity index 100% rename from data/multilingual_humaneval/HumanEval.jsonl rename to mxeval/data/multilingual_humaneval/HumanEval.jsonl diff --git a/data/multilingual_humaneval/HumanEval_csharp_v1.jsonl b/mxeval/data/multilingual_humaneval/HumanEval_csharp_v1.jsonl similarity index 100% rename from data/multilingual_humaneval/HumanEval_csharp_v1.jsonl rename to mxeval/data/multilingual_humaneval/HumanEval_csharp_v1.jsonl diff --git a/data/multilingual_humaneval/HumanEval_go_v1.jsonl b/mxeval/data/multilingual_humaneval/HumanEval_go_v1.jsonl similarity index 100% rename from data/multilingual_humaneval/HumanEval_go_v1.jsonl rename to mxeval/data/multilingual_humaneval/HumanEval_go_v1.jsonl diff --git a/data/multilingual_humaneval/HumanEval_java_v1.1.jsonl b/mxeval/data/multilingual_humaneval/HumanEval_java_v1.1.jsonl similarity index 100% rename from data/multilingual_humaneval/HumanEval_java_v1.1.jsonl rename to mxeval/data/multilingual_humaneval/HumanEval_java_v1.1.jsonl diff --git a/data/multilingual_humaneval/HumanEval_javascript_v1.1.jsonl b/mxeval/data/multilingual_humaneval/HumanEval_javascript_v1.1.jsonl similarity index 100% rename from data/multilingual_humaneval/HumanEval_javascript_v1.1.jsonl rename to mxeval/data/multilingual_humaneval/HumanEval_javascript_v1.1.jsonl diff --git a/data/multilingual_humaneval/HumanEval_kotlin_v1.1.jsonl b/mxeval/data/multilingual_humaneval/HumanEval_kotlin_v1.1.jsonl similarity index 100% rename from data/multilingual_humaneval/HumanEval_kotlin_v1.1.jsonl rename to mxeval/data/multilingual_humaneval/HumanEval_kotlin_v1.1.jsonl diff --git a/data/multilingual_humaneval/HumanEval_perl_v1.1.jsonl b/mxeval/data/multilingual_humaneval/HumanEval_perl_v1.1.jsonl similarity index 100% rename from data/multilingual_humaneval/HumanEval_perl_v1.1.jsonl rename to mxeval/data/multilingual_humaneval/HumanEval_perl_v1.1.jsonl diff --git a/data/multilingual_humaneval/HumanEval_php_v1.1.jsonl b/mxeval/data/multilingual_humaneval/HumanEval_php_v1.1.jsonl similarity index 100% rename from data/multilingual_humaneval/HumanEval_php_v1.1.jsonl rename to mxeval/data/multilingual_humaneval/HumanEval_php_v1.1.jsonl diff --git a/data/multilingual_humaneval/HumanEval_ruby_v1.1.jsonl b/mxeval/data/multilingual_humaneval/HumanEval_ruby_v1.1.jsonl similarity index 100% rename from data/multilingual_humaneval/HumanEval_ruby_v1.1.jsonl rename to mxeval/data/multilingual_humaneval/HumanEval_ruby_v1.1.jsonl diff --git a/data/multilingual_humaneval/HumanEval_scala_v1.1.jsonl b/mxeval/data/multilingual_humaneval/HumanEval_scala_v1.1.jsonl similarity index 100% rename from data/multilingual_humaneval/HumanEval_scala_v1.1.jsonl rename to mxeval/data/multilingual_humaneval/HumanEval_scala_v1.1.jsonl diff --git a/data/multilingual_humaneval/HumanEval_swift_v1.1.jsonl b/mxeval/data/multilingual_humaneval/HumanEval_swift_v1.1.jsonl similarity index 100% rename from data/multilingual_humaneval/HumanEval_swift_v1.1.jsonl rename to mxeval/data/multilingual_humaneval/HumanEval_swift_v1.1.jsonl diff --git a/data/multilingual_humaneval/HumanEval_typescript_v1.jsonl b/mxeval/data/multilingual_humaneval/HumanEval_typescript_v1.jsonl similarity index 100% rename from data/multilingual_humaneval/HumanEval_typescript_v1.jsonl rename to mxeval/data/multilingual_humaneval/HumanEval_typescript_v1.jsonl diff --git a/data/multilingual_humaneval/LICENSE b/mxeval/data/multilingual_humaneval/LICENSE similarity index 100% rename from data/multilingual_humaneval/LICENSE rename to mxeval/data/multilingual_humaneval/LICENSE diff --git a/data/multilingual_humaneval/metadata.json b/mxeval/data/multilingual_humaneval/metadata.json similarity index 100% rename from data/multilingual_humaneval/metadata.json rename to mxeval/data/multilingual_humaneval/metadata.json diff --git a/data/multilingual_mathqa/LICENSE b/mxeval/data/multilingual_mathqa/LICENSE similarity index 100% rename from data/multilingual_mathqa/LICENSE rename to mxeval/data/multilingual_mathqa/LICENSE diff --git a/data/multilingual_mathqa/README.md b/mxeval/data/multilingual_mathqa/README.md similarity index 100% rename from data/multilingual_mathqa/README.md rename to mxeval/data/multilingual_mathqa/README.md diff --git a/data/multilingual_mathqa/examples/java_samples.jsonl b/mxeval/data/multilingual_mathqa/examples/java_samples.jsonl similarity index 100% rename from data/multilingual_mathqa/examples/java_samples.jsonl rename to mxeval/data/multilingual_mathqa/examples/java_samples.jsonl diff --git a/data/multilingual_mathqa/examples/javascript_samples.jsonl b/mxeval/data/multilingual_mathqa/examples/javascript_samples.jsonl similarity index 100% rename from data/multilingual_mathqa/examples/javascript_samples.jsonl rename to mxeval/data/multilingual_mathqa/examples/javascript_samples.jsonl diff --git a/data/multilingual_mathqa/examples/python_samples.jsonl b/mxeval/data/multilingual_mathqa/examples/python_samples.jsonl similarity index 100% rename from data/multilingual_mathqa/examples/python_samples.jsonl rename to mxeval/data/multilingual_mathqa/examples/python_samples.jsonl diff --git a/data/multilingual_mathqa/fewshot_prompting/mathqa_java_v1.java b/mxeval/data/multilingual_mathqa/fewshot_prompting/mathqa_java_v1.java similarity index 100% rename from data/multilingual_mathqa/fewshot_prompting/mathqa_java_v1.java rename to mxeval/data/multilingual_mathqa/fewshot_prompting/mathqa_java_v1.java diff --git a/data/multilingual_mathqa/fewshot_prompting/mathqa_javascript_v1.js b/mxeval/data/multilingual_mathqa/fewshot_prompting/mathqa_javascript_v1.js similarity index 100% rename from data/multilingual_mathqa/fewshot_prompting/mathqa_javascript_v1.js rename to mxeval/data/multilingual_mathqa/fewshot_prompting/mathqa_javascript_v1.js diff --git a/data/multilingual_mathqa/fewshot_prompting/mathqa_python_v1.py b/mxeval/data/multilingual_mathqa/fewshot_prompting/mathqa_python_v1.py similarity index 100% rename from data/multilingual_mathqa/fewshot_prompting/mathqa_python_v1.py rename to mxeval/data/multilingual_mathqa/fewshot_prompting/mathqa_python_v1.py diff --git a/data/multilingual_mathqa/mathqa-test-java_v1.jsonl b/mxeval/data/multilingual_mathqa/mathqa-test-java_v1.jsonl similarity index 100% rename from data/multilingual_mathqa/mathqa-test-java_v1.jsonl rename to mxeval/data/multilingual_mathqa/mathqa-test-java_v1.jsonl diff --git a/data/multilingual_mathqa/mathqa-test-javascript_v1.jsonl b/mxeval/data/multilingual_mathqa/mathqa-test-javascript_v1.jsonl similarity index 100% rename from data/multilingual_mathqa/mathqa-test-javascript_v1.jsonl rename to mxeval/data/multilingual_mathqa/mathqa-test-javascript_v1.jsonl diff --git a/data/multilingual_mathqa/mathqa-test-python_v1.jsonl b/mxeval/data/multilingual_mathqa/mathqa-test-python_v1.jsonl similarity index 100% rename from data/multilingual_mathqa/mathqa-test-python_v1.jsonl rename to mxeval/data/multilingual_mathqa/mathqa-test-python_v1.jsonl diff --git a/data/multilingual_mathqa/metadata.json b/mxeval/data/multilingual_mathqa/metadata.json similarity index 100% rename from data/multilingual_mathqa/metadata.json rename to mxeval/data/multilingual_mathqa/metadata.json diff --git a/setup.py b/setup.py index 6a930f9..b8b378c 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,7 @@ description="", author="AWS AI Labs", packages=find_packages(), + include_package_data=True, install_requires=[ str(r) for r in pkg_resources.parse_requirements( @@ -21,7 +22,7 @@ ], entry_points={ "console_scripts": [ - "evaluate_functional_correctness = mxeval.evaluate_functional_correctness", + "evaluate_functional_correctness = mxeval:evaluate_functional_correctness", ] } )