Skip to content

Commit 35ffd08

Browse files
CasperWACasperWA
authored
Merge pull request #36 from aiidateam/develop
Up to v0.4.0 Due to rebase and commit for v0.3.0, some commits will be redone. This cannot be undone without losing pointers to the v0.3.0 published commit. One should either merge with --ff-only from command line or make a merge commit and then merge back into `develop` using a merge commit as well. This update is considered from commit 12d2237. Updates: - OPTiMaDe spec v0.10.1. - optimade-python-tools v0.3.1 (with grammar for v0.10.1). - Redone HTTP middleware to handle session per request. - Update to pydantic v1. - Add implementation information, i.e., information about which version of `aiida-optimade` the server is using, to all `meta` responses. - Use server.cfg. - Fix broken Docker build, and add it to CI (@ltalirz)
2 parents 60c60b4 + 36c6d7c commit 35ffd08

37 files changed

+1252
-581
lines changed

‎.ci/assert_version.py

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,26 @@
1+
# pylint: disable=line-too-long
12
import os
23
import json
4+
import sys
35
from pathlib import Path
46

57
SETUP_JSON = Path(__file__).resolve().parent.parent.joinpath("setup.json")
68

79
with open(SETUP_JSON, "r") as fp:
8-
setup = json.load(fp)
10+
SETUP = json.load(fp)
911

10-
package_version = "v" + setup["version"]
12+
PACKAGE_VERSION = "v" + SETUP["version"]
1113

12-
tag_version = os.getenv("TAG_VERSION")
13-
tag_version = tag_version[len("refs/tags/") :]
14+
TAG_VERSION = os.getenv("TAG_VERSION")
15+
TAG_VERSION = TAG_VERSION[len("refs/tags/") :]
1416

15-
if tag_version == package_version:
16-
print(f"The versions match: tag:'{tag_version}' == package:'{package_version}'")
17-
exit(0)
17+
if TAG_VERSION == PACKAGE_VERSION:
18+
print(f"The versions match: tag:'{TAG_VERSION}' == package:'{PACKAGE_VERSION}'")
19+
sys.exit(0)
1820

1921
print(
20-
f"""The current package version '{package_version}' does not equal the tag version '{tag_version}'.
22+
f"""The current package version '{PACKAGE_VERSION}' does not equal the tag version '{TAG_VERSION}'.
2123
Update setup.json with new version.
2224
Please remove the tag from both GitHub and your local repository!"""
2325
)
24-
exit(1)
26+
sys.exit(1)

‎.ci/optimade-version.json

Lines changed: 0 additions & 6 deletions
This file was deleted.

‎.ci/optimade_version_update.py

Lines changed: 0 additions & 51 deletions
This file was deleted.

‎server_template.cfg renamed to ‎.ci/server_template.cfg

File renamed without changes.

‎.docker/run.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,10 @@
22
set -ex
33
mkdir -p $AIIDA_PATH/.aiida
44
cp /profiles/$AIIDA_PROFILE.json $AIIDA_PATH/.aiida/config.json
5+
6+
# make docker.host.internal available
7+
# see https://github.com/docker/for-linux/issues/264#issuecomment-387525409
8+
# ltalirz: Only works for Mac, not Linux
9+
# echo -e "`/sbin/ip route|awk '/default/ { print $3 }'`\tdocker.host.internal" | tee -a /etc/hosts > /dev/null
10+
511
uvicorn aiida_optimade.main:app --host 0.0.0.0 --port 80

‎.github/workflows/ci.yml

Lines changed: 62 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ jobs:
1818
- name: Install dependencies
1919
run: |
2020
python -m pip install --upgrade pip
21-
pip install -U setuptools
2221
pip install flake8
2322
2423
- name: Lint with flake8
@@ -34,16 +33,14 @@ jobs:
3433

3534
strategy:
3635
fail-fast: false
37-
matrix:
38-
python-version: [3.7, 3.8]
3936

4037
steps:
4138
- uses: actions/checkout@v1
4239

43-
- name: Set up Python ${{ matrix.python-version}}
40+
- name: Set up Python 3.7
4441
uses: actions/setup-python@v1
4542
with:
46-
python-version: ${{ matrix.python-version}}
43+
python-version: 3.7
4744

4845
- name: Install dependencies
4946
run: |
@@ -52,8 +49,7 @@ jobs:
5249
pip install pre-commit
5350
5451
- name: Test with pre-commit
55-
run: |
56-
pre-commit run --all-files || ( git status --short ; git diff ; exit 1 )
52+
run: pre-commit run --all-files || ( git status --short ; git diff ; exit 1 )
5753

5854
pytest:
5955

@@ -62,7 +58,7 @@ jobs:
6258
strategy:
6359
fail-fast: false
6460
matrix:
65-
python-version: [3.7, 3.8]
61+
python-version: [3.6, 3.7, 3.8]
6662
backend: ['django', 'sqlalchemy']
6763

6864
steps:
@@ -100,21 +96,21 @@ jobs:
10096
pip install -e .[dev]
10197
reentry scan
10298
99+
- name: Setup server.cfg for AiiDA OPTiMaDe server
100+
run: cp .ci/server_template.cfg ./server.cfg
101+
103102
- name: Setup up environment for AiiDA
104103
env:
105104
AIIDA_TEST_BACKEND: ${{ matrix.backend }}
106-
run: |
107-
.github/workflows/setup_aiida.sh
105+
run: .github/workflows/setup_aiida.sh
108106

109107
- name: Load test data
110-
run: |
111-
verdi import --migration --non-interactive .github/aiida/optimade.aiida
108+
run: verdi import --migration --non-interactive .github/aiida/optimade.aiida
112109

113110
- name: Test with pytest
114111
env:
115112
AIIDA_PROFILE: test_${{ matrix.backend }}
116-
run: |
117-
pytest --cov=./aiida_optimade/ --cov-report=xml
113+
run: pytest --cov=./aiida_optimade/ --cov-report=xml
118114

119115
- name: Upload coverage to Codecov
120116
if: matrix.python-version == 3.7
@@ -124,3 +120,55 @@ jobs:
124120
xml: ./coverage.xml
125121
flags: unittests
126122
yml: ./.codecov.yml
123+
124+
docker-image:
125+
runs-on: ubuntu-latest
126+
127+
steps:
128+
- uses: actions/checkout@v1
129+
130+
# - uses: harmon758/postgresql-action@v1
131+
# with:
132+
# postgresql version: '11'
133+
# postgresql db: test_django
134+
# postgresql user: 'postgres'
135+
# postgresql password: ''
136+
137+
# - name: Set up Python 3.7
138+
# uses: actions/setup-python@v1
139+
# with:
140+
# python-version: 3.7
141+
142+
# - name: Install system dependencies
143+
# run: |
144+
# wget -O - "https://github.com/rabbitmq/signing-keys/releases/download/2.0/rabbitmq-release-signing-key.asc" | sudo apt-key add -
145+
# echo 'deb https://dl.bintray.com/rabbitmq-erlang/debian bionic erlang' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list
146+
# echo 'deb https://dl.bintray.com/rabbitmq/debian bionic main' | sudo tee -a /etc/apt/sources.list.d/bintray.rabbitmq.list
147+
# sudo apt update
148+
# sudo apt install postgresql postgresql-server-dev-all postgresql-client rabbitmq-server graphviz
149+
# sudo systemctl status rabbitmq-server.service
150+
151+
# # Install optimade-python-tools from github
152+
# - name: Install python dependencies
153+
# run: |
154+
# cd ${GITHUB_WORKSPACE}/..
155+
# git clone https://github.com/Materials-Consortia/optimade-python-tools
156+
# pip install -e optimade-python-tools
157+
# cd ${GITHUB_WORKSPACE}
158+
# pip install -e .
159+
# reentry scan
160+
161+
# - name: Setup up environment for AiiDA
162+
# env:
163+
# AIIDA_TEST_BACKEND: django
164+
# run: .github/workflows/setup_aiida.sh
165+
166+
# - name: Load test data
167+
# run: verdi import --migration --non-interactive .github/aiida/optimade.aiida
168+
169+
- name: Build the Docker image
170+
run: docker-compose -f profiles/docker-compose.yml build
171+
# .github/workflows/wait_for_it.sh localhost:3253 -t 120
172+
# sleep 15
173+
# curl http://localhost:3253/optimade/info > info.json
174+
# grep -F "www.aiida.net" info.json || exit 1

‎.github/workflows/wait_for_it.sh

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
#!/usr/bin/env bash
2+
# Use this script to test if a given TCP host/port are available
3+
4+
cmdname=$(basename $0)
5+
6+
echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
7+
8+
usage()
9+
{
10+
cat << USAGE >&2
11+
Usage:
12+
$cmdname host:port [-s] [-t timeout] [-- command args]
13+
-h HOST | --host=HOST Host or IP under test
14+
-p PORT | --port=PORT TCP port under test
15+
Alternatively, you specify the host and port as host:port
16+
-s | --strict Only execute subcommand if the test succeeds
17+
-q | --quiet Don't output any status messages
18+
-t TIMEOUT | --timeout=TIMEOUT
19+
Timeout in seconds, zero for no timeout
20+
-- COMMAND ARGS Execute command with args after the test finishes
21+
USAGE
22+
exit 1
23+
}
24+
25+
wait_for()
26+
{
27+
if [[ $TIMEOUT -gt 0 ]]; then
28+
echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT"
29+
else
30+
echoerr "$cmdname: waiting for $HOST:$PORT without a timeout"
31+
fi
32+
start_ts=$(date +%s)
33+
while :
34+
do
35+
(echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1
36+
result=$?
37+
if [[ $result -eq 0 ]]; then
38+
end_ts=$(date +%s)
39+
echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds"
40+
break
41+
fi
42+
sleep 1
43+
done
44+
return $result
45+
}
46+
47+
wait_for_wrapper()
48+
{
49+
# In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
50+
if [[ $QUIET -eq 1 ]]; then
51+
timeout $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
52+
else
53+
timeout $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
54+
fi
55+
PID=$!
56+
trap "kill -INT -$PID" INT
57+
wait $PID
58+
RESULT=$?
59+
if [[ $RESULT -ne 0 ]]; then
60+
echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT"
61+
fi
62+
return $RESULT
63+
}
64+
65+
# process arguments
66+
while [[ $# -gt 0 ]]
67+
do
68+
case "$1" in
69+
*:* )
70+
hostport=(${1//:/ })
71+
HOST=${hostport[0]}
72+
PORT=${hostport[1]}
73+
shift 1
74+
;;
75+
--child)
76+
CHILD=1
77+
shift 1
78+
;;
79+
-q | --quiet)
80+
QUIET=1
81+
shift 1
82+
;;
83+
-s | --strict)
84+
STRICT=1
85+
shift 1
86+
;;
87+
-h)
88+
HOST="$2"
89+
if [[ $HOST == "" ]]; then break; fi
90+
shift 2
91+
;;
92+
--host=*)
93+
HOST="${1#*=}"
94+
shift 1
95+
;;
96+
-p)
97+
PORT="$2"
98+
if [[ $PORT == "" ]]; then break; fi
99+
shift 2
100+
;;
101+
--port=*)
102+
PORT="${1#*=}"
103+
shift 1
104+
;;
105+
-t)
106+
TIMEOUT="$2"
107+
if [[ $TIMEOUT == "" ]]; then break; fi
108+
shift 2
109+
;;
110+
--timeout=*)
111+
TIMEOUT="${1#*=}"
112+
shift 1
113+
;;
114+
--)
115+
shift
116+
CLI="$@"
117+
break
118+
;;
119+
--help)
120+
usage
121+
;;
122+
*)
123+
echoerr "Unknown argument: $1"
124+
usage
125+
;;
126+
esac
127+
done
128+
129+
if [[ "$HOST" == "" || "$PORT" == "" ]]; then
130+
echoerr "Error: you need to provide a host and port to test."
131+
usage
132+
fi
133+
134+
TIMEOUT=${TIMEOUT:-15}
135+
STRICT=${STRICT:-0}
136+
CHILD=${CHILD:-0}
137+
QUIET=${QUIET:-0}
138+
139+
if [[ $CHILD -gt 0 ]]; then
140+
wait_for
141+
RESULT=$?
142+
exit $RESULT
143+
else
144+
if [[ $TIMEOUT -gt 0 ]]; then
145+
wait_for_wrapper
146+
RESULT=$?
147+
else
148+
wait_for
149+
RESULT=$?
150+
fi
151+
fi
152+
153+
if [[ $CLI != "" ]]; then
154+
if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then
155+
echoerr "$cmdname: strict mode, refusing to execute subprocess"
156+
exit $RESULT
157+
fi
158+
exec $CLI
159+
else
160+
exit $RESULT
161+
fi

0 commit comments

Comments
 (0)