Skip to content

Commit 5c89ed1

Browse files
thomwolfthomwolf
authored
Benchmarks (huggingface#535)
* adding benchmarks * setting up dvc * add pipeline * add results folder * formating metric a little * update * update pipeline and style * sort columns * add benchmark on indices mapping * code quality * update black to new version
1 parent 78ee759 commit 5c89ed1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1218
-791
lines changed

‎.circleci/config.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@ jobs:
2323
# we need a version of isort with https://github.com/timothycrosley/isort/pull/1000
2424
- run: sudo pip install git+git://github.com/timothycrosley/isort.git@e63ae06ec7d70b06df9e528357650281a3d3ec22#egg=isort
2525
- run: sudo pip install .[quality]
26-
- run: black --check --line-length 119 --target-version py36 tests src
27-
- run: isort --check-only --recursive tests src
28-
- run: flake8 tests src
26+
- run: black --check --line-length 119 --target-version py36 src tests benchmarks
27+
- run: isort --check-only --recursive src tests benchmarks
28+
- run: flake8 src tests benchmarks
2929
build_doc:
3030
working_directory: ~/nlp
3131
docker:

‎.dvc/.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
/config.local
2+
/tmp
3+
/cache

‎.dvc/config

Whitespace-only changes.

‎.dvc/plots/confusion.json

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
3+
"data": {
4+
"values": "<DVC_METRIC_DATA>"
5+
},
6+
"title": "<DVC_METRIC_TITLE>",
7+
"mark": "rect",
8+
"encoding": {
9+
"x": {
10+
"field": "<DVC_METRIC_X>",
11+
"type": "nominal",
12+
"sort": "ascending",
13+
"title": "<DVC_METRIC_X_LABEL>"
14+
},
15+
"y": {
16+
"field": "<DVC_METRIC_Y>",
17+
"type": "nominal",
18+
"sort": "ascending",
19+
"title": "<DVC_METRIC_Y_LABEL>"
20+
},
21+
"color": {
22+
"aggregate": "count",
23+
"type": "quantitative"
24+
},
25+
"facet": {
26+
"field": "rev",
27+
"type": "nominal"
28+
}
29+
}
30+
}

‎.dvc/plots/default.json

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
3+
"data": {
4+
"values": "<DVC_METRIC_DATA>"
5+
},
6+
"title": "<DVC_METRIC_TITLE>",
7+
"mark": {
8+
"type": "line"
9+
},
10+
"encoding": {
11+
"x": {
12+
"field": "<DVC_METRIC_X>",
13+
"type": "quantitative",
14+
"title": "<DVC_METRIC_X_LABEL>"
15+
},
16+
"y": {
17+
"field": "<DVC_METRIC_Y>",
18+
"type": "quantitative",
19+
"title": "<DVC_METRIC_Y_LABEL>",
20+
"scale": {
21+
"zero": false
22+
}
23+
},
24+
"color": {
25+
"field": "rev",
26+
"type": "nominal"
27+
}
28+
}
29+
}

‎.dvc/plots/scatter.json

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
3+
"data": {
4+
"values": "<DVC_METRIC_DATA>"
5+
},
6+
"title": "<DVC_METRIC_TITLE>",
7+
"mark": "point",
8+
"encoding": {
9+
"x": {
10+
"field": "<DVC_METRIC_X>",
11+
"type": "quantitative",
12+
"title": "<DVC_METRIC_X_LABEL>"
13+
},
14+
"y": {
15+
"field": "<DVC_METRIC_Y>",
16+
"type": "quantitative",
17+
"title": "<DVC_METRIC_Y_LABEL>",
18+
"scale": {
19+
"zero": false
20+
}
21+
},
22+
"color": {
23+
"field": "rev",
24+
"type": "nominal"
25+
}
26+
}
27+
}

‎.dvc/plots/smooth.json

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"$schema": "https://vega.github.io/schema/vega-lite/v4.json",
3+
"data": {
4+
"values": "<DVC_METRIC_DATA>"
5+
},
6+
"title": "<DVC_METRIC_TITLE>",
7+
"mark": {
8+
"type": "line"
9+
},
10+
"encoding": {
11+
"x": {
12+
"field": "<DVC_METRIC_X>",
13+
"type": "quantitative",
14+
"title": "<DVC_METRIC_X_LABEL>"
15+
},
16+
"y": {
17+
"field": "<DVC_METRIC_Y>",
18+
"type": "quantitative",
19+
"title": "<DVC_METRIC_Y_LABEL>",
20+
"scale": {
21+
"zero": false
22+
}
23+
},
24+
"color": {
25+
"field": "rev",
26+
"type": "nominal"
27+
}
28+
},
29+
"transform": [
30+
{
31+
"loess": "<DVC_METRIC_Y>",
32+
"on": "<DVC_METRIC_X>",
33+
"groupby": [
34+
"rev"
35+
],
36+
"bandwidth": 0.3
37+
}
38+
]
39+
}

‎.dvcignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Add patterns of files dvc should ignore, which could improve
2+
# the performance. Learn more at
3+
# https://dvc.org/doc/user-guide/dvcignore

‎.github/workflows/benchmarks.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: benchmarks
2+
on: [push]
3+
jobs:
4+
run:
5+
runs-on: [ubuntu-latest]
6+
container: docker://dvcorg/cml-py3:latest
7+
steps:
8+
- uses: actions/checkout@v2
9+
- name: cml_run
10+
env:
11+
repo_token: ${{ secrets.GITHUB_TOKEN }}
12+
run: |
13+
# Your ML workflow goes here
14+
pip install -e .[tests]
15+
dvc repro
16+
17+
git fetch --prune
18+
dvc metrics diff --show-json master > report.json
19+
20+
python ./benchmarks/format.py report.json report.md
21+
22+
cml-send-comment report.md

‎.gitignore

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,9 @@ venv.bak/
5454
# Sphinx documentation
5555
docs/_build/
5656
docs/source/_build/
57+
58+
# Benchmark results
59+
/benchmarks/results/*
60+
!/benchmarks/results/.gitkeep
61+
report.json
62+
report.md

0 commit comments

Comments
 (0)