Skip to content

Commit 9c5ec03

Browse files
mattjalamattjala
committed
Update scripts to use docker compose v2
1 parent 8ff4791 commit 9c5ec03

File tree

11 files changed

+77
-28
lines changed

11 files changed

+77
-28
lines changed

‎.github/workflows/python-package.yml

Lines changed: 61 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
matrix:
1616
os: [ubuntu-latest, windows-latest]
1717
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
18-
test-other: ["none", "h5pyd"]
18+
build-method: ["manual", "docker"]
1919

2020
runs-on: ${{ matrix.os }}
2121
steps:
@@ -49,9 +49,9 @@ jobs:
4949
shell: bash
5050
run: |
5151
pytest
52-
53-
- name: Run integration tests
52+
- name: Start HSDS (Manual)
5453
shell: bash
54+
if: ${{matrix.build-method == 'manual'}}
5555
env:
5656
ADMIN_PASSWORD: admin
5757
ADMIN_USERNAME: admin
@@ -60,52 +60,101 @@ jobs:
6060
USER2_NAME: test_user2
6161
USER2_PASSWORD: test
6262
HSDS_ENDPOINT: http://127.0.0.1:5101
63-
63+
BUCKET_NAME: hsdstest
6464
run: |
6565
mkdir data
6666
mkdir data/hsdstest
6767
cp admin/config/groups.default admin/config/groups.txt
6868
cp admin/config/passwd.default admin/config/passwd.txt
6969
hsds --root_dir data --host localhost --port 5101 --password_file admin/config/passwd.txt --logfile hs.log --loglevel DEBUG --config_dir=admin/config --count=4 &
70-
sleep 10 # let the nodes get ready
70+
71+
- name: Configure Docker (Windows)
72+
if: ${{(matrix.os == 'windows-latest') && (matrix.build-method == 'docker')}}
73+
shell: bash
74+
run: |
75+
cd "D:\"
76+
dir
77+
78+
- name: Start HSDS (Docker)
79+
shell: bash
80+
if: ${{matrix.build-method == 'docker'}}
81+
env:
82+
ADMIN_PASSWORD: admin
83+
ADMIN_USERNAME: admin
84+
USER_NAME: test_user1
85+
USER_PASSWORD: test
86+
USER2_NAME: test_user2
87+
USER2_PASSWORD: test
88+
HSDS_ENDPOINT: http://127.0.0.1:5101
89+
ROOT_DIR: ${{github.workspace}}/data
90+
BUCKET_NAME: hsdstest
91+
run: |
92+
mkdir data
93+
mkdir data/hsdstest
94+
cp admin/config/groups.default admin/config/groups.txt
95+
cp admin/config/passwd.default admin/config/passwd.txt
96+
./build.sh --nolint
97+
./runall.sh
98+
99+
- name: Wait for node startup
100+
shell: bash
101+
run: |
102+
sleep 30
103+
104+
- name: Run HSDS tests
105+
shell: bash
106+
env:
107+
ADMIN_PASSWORD: admin
108+
ADMIN_USERNAME: admin
109+
USER_NAME: test_user1
110+
USER_PASSWORD: test
111+
USER2_NAME: test_user2
112+
USER2_PASSWORD: test
113+
BUCKET_NAME: hsdstest
114+
run: |
71115
pytest tests/integ/setup_test.py
72116
pytest tests/integ
73117
74118
- name: Checkout h5pyd
75-
if: ${{ ( matrix.test-other == 'h5pyd' ) && ( matrix.os != 'windows-latest' ) }}
119+
if: ${{ ( matrix.os != 'windows-latest' ) }}
76120
uses: actions/checkout@v4
77121
with:
78122
repository: HDFGroup/h5pyd
79123
path: ${{github.workspace}}/h5pyd
80124

81125
- name: Install h5pyd
82-
if: ${{ ( matrix.test-other == 'h5pyd' ) && ( matrix.os != 'windows-latest' ) }}
126+
if: ${{ ( matrix.os != 'windows-latest' ) }}
83127
working-directory: ${{github.workspace}}/h5pyd
84128
run: |
85129
pip install . -v
86130
87131
- name: Create h5pyd test folder
88-
if: ${{ ( matrix.test-other == 'h5pyd' ) && ( matrix.os != 'windows-latest' ) }}
132+
if: ${{ ( matrix.os != 'windows-latest' ) }}
89133
working-directory: ${{github.workspace}}/h5pyd
90134
env:
91135
HS_USERNAME: test_user1
92136
HS_PASSWORD: test
93137
TEST2_USERNAME: test_user1
94138
TEST2_PASSWORD: test
95-
HS_ENDPOINT: http://127.0.0.1:5101
96139
H5PYD_TEST_FOLDER: /home/test_user1/h5pyd_tests/
140+
HS_ENDPOINT: http://127.0.0.1:5101
97141
run: |
98-
hstouch -e http://127.0.0.1:5101 /home/test_user1/h5pyd_tests/
142+
hstouch -e $HS_ENDPOINT /home/test_user1/h5pyd_tests/
99143
100144
- name: Run h5pyd tests
101-
if: ${{ ( matrix.test-other == 'h5pyd' ) && ( matrix.os != 'windows-latest' ) }}
145+
if: ${{ ( matrix.os != 'windows-latest' ) }}
102146
working-directory: ${{github.workspace}}/h5pyd
103147
env:
104148
HS_USERNAME: test_user1
105149
HS_PASSWORD: test
106150
TEST2_USERNAME: test_user1
107151
TEST2_PASSWORD: test
108-
HS_ENDPOINT: http://127.0.0.1:5101
109152
H5PYD_TEST_FOLDER: /home/test_user1/h5pyd_tests/
153+
HS_ENDPOINT: http://127.0.0.1:5101
110154
run: |
111155
python testall.py
156+
157+
- name: Shut down Docker
158+
if: ${{matrix.build-method == 'docker'}}
159+
run: |
160+
./stopall.sh

‎docs/design/azure/azure.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ The current implementation of HSDS uses these AWS specific technologies:
2828
- AWS S3 - Storage of HDF data
2929
- AWS DynamoDB - HSDS usernames and passwords
3030

31-
When running HSDS on a single VM using Docker, there should be no differences between AWS and Azure as far as how containers are set (i.e. the same docker-compose configuration should work for both).
31+
When running HSDS on a single VM using Docker, there should be no differences between AWS and Azure as far as how containers are set (i.e. the same docker compose configuration should work for both).
3232

3333
For running HSDS in a self-managed Kubernetes cluster, this should also work the same way on Azure. In addition, both Amazon and Microsoft offers a supported Kubernetes services: AWS EKS and Azure AKS. But self-hosted Kubernetes, vs AWS EKS vs. Azure should have minimal differences given common Kubernetes API and command line tools (e.g. kubectl), so we won't address Kubernetes differences here.
3434

‎docs/docker_install_aws.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ Installation using Docker with AWS S3 storage
44
Export environment variables as shown in "Sample .bashrc" below.
55

66
1. Install Python 3 (e.g. with Miniconda <https://docs.conda.io/en/latest/miniconda.html>)
7-
2. Install Docker and docker-compose if necessary. See [Docker Setup](setup_docker.md)
7+
2. Install Docker and docker compose if necessary. See [Docker Setup](setup_docker.md)
88
3. Create a bucket for HSDS, using aws cli tools or aws management console
99
4. Get project source code: `$ git clone https://github.com/HDFGroup/hsds`
1010
5. Build the docker image: `$ ./build.sh --nolint`

‎docs/docker_install_azure.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ These environment variables will be passed to the Docker containers on startup.
6868
Follow the following steps to setup HSDS:
6969

7070
1. SSH to the VM created above. Replace [publicIpAddress] with the public IP displayed in the output of your VM creation command above: `ssh $VM_USER@[publicIpAddress]`
71-
2. Install Docker and docker-compose if necessary. See [Docker Setup](setup_docker.md)
71+
2. Install Docker and docker compose if necessary. See [Docker Setup](setup_docker.md)
7272
3. Get project source code: `git clone https://github.com/HDFGroup/hsds`
7373
4. If you plan to use HTTP Basic Auth (usernames and passwords managed by the service), go to hsds/admin/config directory: `cd admin/config`, and copy the file "passwd.default" to "passwd.txt". Add any usernames/passwords you wish. Modify existing passwords (for admin, test_user1, test_user2, etc.) for security. If you wish to use Azure Active Directory for authentication, follow the instructions in [Azure Active Directory Setup](azure_ad_setup.md)
7474
5. If group-level permissions are desired (See [Authorization](authorization.md)), copy the file "groups.default" to "groups.txt". Modify existing groups as needed

‎docs/docker_install_posix.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ Export environment variables as shown in "Sample .bashrc" below.
99
2. Create a directory for storage files and set the ROOT_DIR environment variable to point to it
1010
3. Create a subdirectory under ROOT_DIR that will be the default location when "bucket" is not defined.
1111
4. Set the environment variable BUCKET_NAME to the name of the subdirectory
12-
5. Install Docker and docker-compose if necessary. See [Docker Setup](setup_docker.md)
12+
5. Install Docker and docker compose if necessary. See [Docker Setup](setup_docker.md)
1313
6. Get project source code: `$ git clone https://github.com/HDFGroup/hsds`
1414
7. Go to admin/config directory: `$ cd hsds/admin/config`
1515
8. Copy the file "passwd.default" to "passwd.txt". Add any usernames/passwords you wish. Modify existing passwords (for admin, test_user1, test_user2) for security
@@ -53,5 +53,5 @@ To get the latest codes changes from the HSDS repo do the following:
5353

5454
1. Shutdown the service: `$ stopall.sh`
5555
2. Get code changes: `$ git pull`
56-
3. Build a new Docker image: `$ docker-compose build`
56+
3. Build a new Docker image: `$ docker compose build`
5757
4. Start the service: `$ ./runall.sh`

‎docs/docker_install_tencent.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ values should be fine. Record the bucket name and endpoint value.
1313
8. For the security group, enable ports 22, and 80
1414
9. Start instance and ssh into the instance. The following steps will be performed on the instance
1515
10. Install Python 3 (e.g. with Miniconda <https://docs.conda.io/en/latest/miniconda.html>)
16-
11. Install docker-compose by running: `$ curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose`, and `$ chmod +x /usr/local/bin/docker-compose`
16+
11. Install docker compose by running: `$ curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker compose`, and `$ chmod +x /usr/local/bin/docker-compose`
1717
12. If docker is not running, start it with: `systemctl start docker`
1818
13. Get project source code: `$ git clone https://github.com/HDFGroup/hsds`
1919
14. Go to the hsds directory and run: `$ ./build.sh`

‎docs/setup_docker.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ Run the following commands to install Docker on Linux/Ubuntu:
1616
7. Log out and back in again (you may also need to stop/start docker service)
1717
8. `docker ps` to verify that Docker is running.
1818

19-
Install docker-compose:
19+
Install docker compose:
2020

2121
1. See: <https://docs.docker.com/compose/install/>
2222

‎runall.sh

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -132,8 +132,8 @@ if [[ ${NO_DOCKER} ]]; then
132132
echo "using password file: ${PASSWORD_FILE}"
133133

134134
else
135-
# check that docker-compose is available
136-
docker-compose --version >/dev/null || exit 1
135+
# check that docker compose is available
136+
docker compose --version >/dev/null || exit 1
137137
if [[ -z ${COMPOSE_PROJECT_NAME} ]]; then
138138
export COMPOSE_PROJECT_NAME=hsds # use "hsds_" as prefix for container names
139139
fi
@@ -202,12 +202,12 @@ if [[ $NO_DOCKER ]] ; then
202202
else
203203
if [[ $DOCKER_CMD == "down" ]]; then
204204
# use the compose file to shutdown the sevice
205-
echo "Running docker-compose -f ${COMPOSE_FILE} down"
206-
docker-compose -f ${COMPOSE_FILE} down
205+
echo "Running docker compose -f ${COMPOSE_FILE} down"
206+
docker compose -f ${COMPOSE_FILE} down
207207
exit 0 # can quit now
208208
else
209-
echo "Running docker-compose -f ${COMPOSE_FILE} up -d --scale sn=${SN_CORES} --scale dn=${DN_CORES}"
210-
docker-compose -f ${COMPOSE_FILE} up -d --scale sn=${SN_CORES} --scale dn=${DN_CORES}
209+
echo "Running docker compose -f ${COMPOSE_FILE} up -d --scale sn=${SN_CORES} --scale dn=${DN_CORES}"
210+
docker compose -f ${COMPOSE_FILE} up -d --scale sn=${SN_CORES} --scale dn=${DN_CORES}
211211
fi
212212

213213
# wait for the server to be ready

‎stopall.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

33
#
4-
# Shutdown HSDS with "docker-compose down" using the appropriate compose file
4+
# Shutdown HSDS with "docker compose down" using the appropriate compose file
55
#
66
./runall.sh --stop

‎tests/perf/write/start.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,4 @@ fi
2727

2828
echo "count $count"
2929

30-
docker-compose -f docker-compose.yml up -d --scale hswritetest=$count
30+
docker compose -f docker-compose.yml up -d --scale hswritetest=$count

0 commit comments

Comments
 (0)