From 999c49910a7a2396f241bd6f933dc1af9e7df2d3 Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Fri, 23 Aug 2024 15:35:46 +0100 Subject: [PATCH 01/20] Point out that the playbooks work on CentOS 7 only, remove ome-ansible-molecule --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 798e3c28..1405bc34 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ OME production services playbooks ================================= +**NOTE: We are in the process of migration of the playbooks in this repo to Rocky Linux 9/RHEL 9 OS but at the moment, they function only on CentOS 7 !!** These playbooks encapsulate the running of various production servers run by the OME team. If you are looking for examples of running your own production OMERO.server see @@ -10,7 +11,7 @@ If you are looking for examples of running your own production OMERO.server see Details ------- -- Install `Ansible` and dependencies using the [ome-ansible-molecule package](https://pypi.org/project/ome-ansible-molecule/). +- Install `Ansible` >2.10. - Install required roles: `ansible-galaxy install -r requirements.yml` - Run the [`site.yml` playbook](site.yml). From feec0b310031a8ffa60368c915a9add3f035b5bd Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Mon, 26 Aug 2024 16:45:46 +0100 Subject: [PATCH 02/20] Adjust demo playbook,requirements.yml, delete the rest, adjust README --- .github/workflows/molecule.yml | 11 +- README.md | 21 +- Vagrantfile | 24 - ansible.cfg | 2 +- bootstrap/playbook.yml | 36 -- k8s/README.md | 17 - k8s/bootstrap/playbook.yml | 5 - k8s/prerequisites/playbook.yml | 6 - molecule/bootstrap/molecule.yml | 28 - molecule/docker-prod/Dockerfile.j2 | 22 - molecule/docker-prod/converge.yml | 19 - molecule/docker-prod/molecule.yml | 56 -- molecule/docker-prod/prepare.yml | 22 - molecule/docker-prod/tests/test_default.py | 39 -- molecule/nightshade-webclients/molecule.yml | 58 -- .../tests/test_default.py | 40 -- molecule/ome-demoserver/molecule.yml | 40 +- molecule/ome-dundeeomero/Dockerfile.j2 | 22 - molecule/ome-dundeeomero/molecule.yml | 59 -- .../ome-dundeeomero/tests/test_default.py | 46 -- molecule/ome-pg-prod/molecule_disabled.yml | 28 - molecule/ome-pg-prod/prepare.yml | 9 - molecule/ome-pg-prod/tests/test_default.py | 16 - molecule/omero-training-server/Dockerfile.j2 | 22 - molecule/omero-training-server/molecule.yml | 59 -- .../tests/test_default.py | 63 --- molecule/prepare.yml | 12 + molecule/release/molecule.yml | 37 -- molecule/release/prepare.yml | 72 --- molecule/release/tests/test_default.py | 49 -- molecule/resources/Dockerfile.j2 | 22 - molecule/resources/prepare-iproute.yml | 15 - molecule/web-proxy/molecule.yml | 30 - molecule/www/Dockerfile.j2 | 1 - molecule/www/molecule.yml | 27 - molecule/www/tests/test_default.py | 25 - molecule/www/tests/test_redirects.py | 116 ---- omedev/docker-prod-apps.yml | 349 ------------ omedev/playbook.yml | 16 - omero/learning.yml | 201 ------- omero/nightshade-webclients.yml | 185 ------- omero/ome-demoserver.yml | 178 +----- omero/ome-dundeeomero.yml | 278 ---------- omero/omero-firewall.yml | 68 --- omero/omero-monitoring-agents.yml | 115 ---- omero/roles/.gitignore | 2 + omero/sls-gallery.yml | 171 ------ .../templates/omero-web-config-for-webapps.j2 | 6 +- omero/training-server/idr_data.yml | 24 - omero/training-server/letsencrypt.yml | 72 --- .../maintenance/omero-restoredb.yml | 81 --- omero/training-server/playbook.yml | 511 ------------------ omero/training-server/templates | 1 - postgres/ome-pg-prod.yml | 31 -- release/group_vars/all.yml | 67 --- release/permissions.yml | 37 -- release/presentations.yml | 13 - release/public-images.yml | 25 - release/release-acceptance.yml | 97 ---- requirements.yml | 115 +--- site.yml | 34 -- vendor/.gitignore | 1 - web-proxy/playbook.yml | 33 -- www/README.txt | 7 - www/Vagrantfile | 14 - www/files/deploy | 85 --- www/playbook.yml | 116 ---- www/tests/check_redirects.py | 179 ------ www/tests/requirements.txt | 2 - www/www-deploy.yml | 350 ------------ www/www-static.yml | 82 --- 71 files changed, 118 insertions(+), 4604 deletions(-) delete mode 100644 Vagrantfile delete mode 100644 bootstrap/playbook.yml delete mode 100644 k8s/README.md delete mode 100644 k8s/bootstrap/playbook.yml delete mode 100644 k8s/prerequisites/playbook.yml delete mode 100644 molecule/bootstrap/molecule.yml delete mode 100644 molecule/docker-prod/Dockerfile.j2 delete mode 100644 molecule/docker-prod/converge.yml delete mode 100644 molecule/docker-prod/molecule.yml delete mode 100644 molecule/docker-prod/prepare.yml delete mode 100644 molecule/docker-prod/tests/test_default.py delete mode 100644 molecule/nightshade-webclients/molecule.yml delete mode 100644 molecule/nightshade-webclients/tests/test_default.py delete mode 100644 molecule/ome-dundeeomero/Dockerfile.j2 delete mode 100644 molecule/ome-dundeeomero/molecule.yml delete mode 100644 molecule/ome-dundeeomero/tests/test_default.py delete mode 100644 molecule/ome-pg-prod/molecule_disabled.yml delete mode 100644 molecule/ome-pg-prod/prepare.yml delete mode 100644 molecule/ome-pg-prod/tests/test_default.py delete mode 100644 molecule/omero-training-server/Dockerfile.j2 delete mode 100644 molecule/omero-training-server/molecule.yml delete mode 100644 molecule/omero-training-server/tests/test_default.py create mode 100644 molecule/prepare.yml delete mode 100644 molecule/release/molecule.yml delete mode 100644 molecule/release/prepare.yml delete mode 100644 molecule/release/tests/test_default.py delete mode 100644 molecule/resources/Dockerfile.j2 delete mode 100644 molecule/resources/prepare-iproute.yml delete mode 100644 molecule/web-proxy/molecule.yml delete mode 120000 molecule/www/Dockerfile.j2 delete mode 100644 molecule/www/molecule.yml delete mode 100644 molecule/www/tests/test_default.py delete mode 100644 molecule/www/tests/test_redirects.py delete mode 100644 omedev/docker-prod-apps.yml delete mode 100644 omedev/playbook.yml delete mode 100644 omero/learning.yml delete mode 100644 omero/nightshade-webclients.yml delete mode 100644 omero/ome-dundeeomero.yml delete mode 100644 omero/omero-firewall.yml delete mode 100644 omero/omero-monitoring-agents.yml create mode 100644 omero/roles/.gitignore delete mode 100644 omero/sls-gallery.yml delete mode 100644 omero/training-server/idr_data.yml delete mode 100644 omero/training-server/letsencrypt.yml delete mode 100644 omero/training-server/maintenance/omero-restoredb.yml delete mode 100644 omero/training-server/playbook.yml delete mode 120000 omero/training-server/templates delete mode 100644 postgres/ome-pg-prod.yml delete mode 100644 release/group_vars/all.yml delete mode 100644 release/permissions.yml delete mode 100644 release/presentations.yml delete mode 100644 release/public-images.yml delete mode 100644 release/release-acceptance.yml delete mode 100644 site.yml delete mode 100644 vendor/.gitignore delete mode 100644 web-proxy/playbook.yml delete mode 100644 www/README.txt delete mode 100644 www/Vagrantfile delete mode 100644 www/files/deploy delete mode 100644 www/playbook.yml delete mode 100644 www/tests/check_redirects.py delete mode 100644 www/tests/requirements.txt delete mode 100644 www/www-deploy.yml delete mode 100644 www/www-static.yml diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index a422025a..8642c0f2 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -8,11 +8,11 @@ on: jobs: list-scenarios: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: matrix: ${{ steps.listscenarios.outputs.scenarios }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - id: listscenarios uses: ome/action-ansible-molecule-list-scenarios@main @@ -20,22 +20,21 @@ jobs: name: Test needs: - list-scenarios - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: # Keep running so we can see if other tests pass fail-fast: false matrix: scenario: ${{fromJson(needs.list-scenarios.outputs.matrix)}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: '3.9' - name: Install Ansible & Molecule run: | pip install "ansible<8" "ansible-lint<6.13" flake8 pip install "molecule<5" "ansible-compat<4" pip install molecule-plugins[docker] pytest-testinfra - pip3.8 install jmespath - name: Run molecule run: molecule test -s "${{ matrix.scenario }}" diff --git a/README.md b/README.md index 1405bc34..903bab7c 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ OME production services playbooks ================================= -**NOTE: We are in the process of migration of the playbooks in this repo to Rocky Linux 9/RHEL 9 OS but at the moment, they function only on CentOS 7 !!** These playbooks encapsulate the running of various production servers run by the OME team. + +At the moment, only the `ome-demoserver.yml` playbook is available here. This is a playbook for OMERO.demo server on https://demo.openmicroscopy.org OMERO.server and OMERO.web. You can read more about this [OMERO.demo server on our website](https://www.openmicroscopy.org/explore/). + +We are in the process of adding more OME team's production playbooks here. + If you are looking for examples of running your own production OMERO.server see https://github.com/ome/omero-deployment-examples @@ -11,13 +15,20 @@ If you are looking for examples of running your own production OMERO.server see Details ------- -- Install `Ansible` >2.10. +- Install `Ansible` >2.10 - Install required roles: `ansible-galaxy install -r requirements.yml` -- Run the [`site.yml` playbook](site.yml). +- Run the `ome-demoserver.yml` playbook: + +``` +cd omero +ansible-playbook --ask-become --become -i $PATH/TO/INVENTORY ome-demoserver.yml -l $YOUR-HOST-ADDRESS-OR-IP --diff +``` + -For details of individual playbooks see the comments in [`site.yml`](site.yml). Testing ------- -All server playbooks have a corresponding [molecule](https://molecule.readthedocs.io/) test scenario under [`molecule`](molecule). +We test the playbooks here on Rocky Linux 9 platform via [Ansible Molecule](https://molecule.readthedocs.io/), see test scenarios under [`molecule`](molecule). + +The main components of the playbooks (roles) are being independently tested on both Rocky Linux 9 and Ubuntu 22.04. See e.g. [ome.omero_server role](https://github.com/ome/ansible-role-omero-server/tree/master/molecule). diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index fd052344..00000000 --- a/Vagrantfile +++ /dev/null @@ -1,24 +0,0 @@ -Vagrant.configure("2") do |config| - config.vm.box = "centos/7" - config.vm.provider "virtualbox" do |vb| - vb.customize ["modifyvm", :id, "--memory", "2048"] - config.vm.network "forwarded_port", guest: 80, host: 8080 - config.vm.network "forwarded_port", guest: 4064, host: 4064 - config.vm.network "forwarded_port", guest: 4063, host: 4063 - end - - [ - "ome-dundeeomero", - "ome-demoserver", - "nightshade-web" - ].each do |server| - config.vm.define "#{server}" do |node| - node.vm.box = "centos/7" - node.vm.provision "ansible" do |ansible| - ansible.playbook = "tests/#{server}.yml" - config.vm.provision "ansible" do |ansible| - ansible.skip_tags = "lvm" - ansible.playbook = "#{server}.yml" - ansible.galaxy_role_file = "requirements.yml" - end -end diff --git a/ansible.cfg b/ansible.cfg index 50758f18..67e073b8 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -10,7 +10,7 @@ [defaults] # Galaxy roles -roles_path = ./vendor +roles_path = ./omero/roles # These tend to be annoying. retry_files_enabled = False diff --git a/bootstrap/playbook.yml b/bootstrap/playbook.yml deleted file mode 100644 index 1dc43c86..00000000 --- a/bootstrap/playbook.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -# Playbook which runs the necessary root-level steps -# so that a host can be managed by others -- name: Boot1 - hosts: omedev - roles: - - - role: ome.network - - - role: ome.lvm_partition - lvm_lvname: var_log - lvm_lvmount: /var/log - lvm_lvsize: 4g - lvm_lvfilesystem: xfs - lvm_vgname: VolGroup00 - - role: ome.lvm_partition - lvm_lvname: root - lvm_lvmount: / - lvm_lvsize: 100%FREE - lvm_lvfilesystem: xfs - lvm_vgname: VolGroup00 - lvm_shrink: false - - - role: ome.sudoers - sudoers_individual_commands: - - user: "%omedev" - become: ALL - command: "NOPASSWD: ALL" - - - role: ome.upgrade_distpackages - upgrade_distpackages_reboot_kernel: true - -- name: Network - hosts: vlan-10ge-servers, vlan-ome-idr-docker - roles: - - role: ome.network diff --git a/k8s/README.md b/k8s/README.md deleted file mode 100644 index 554016f2..00000000 --- a/k8s/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# OME kubernetes suport playbooks - -## `bootstrap` - -Provisioning tasks intended to be run once when provisioning a new system. -This includes networking configuration. - - -## `prerequisites` - -These tasks should be run before a Kubernetes cluster is promoted to production use. -It should be safe to re-run these playbooks at any time. - - -## `postgres` - -An standalone PostgreSQL server for use by Kubernetes applications. diff --git a/k8s/bootstrap/playbook.yml b/k8s/bootstrap/playbook.yml deleted file mode 100644 index c31b457f..00000000 --- a/k8s/bootstrap/playbook.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: Bootkub - hosts: vlan-10ge-servers - roles: - - role: ome.network diff --git a/k8s/prerequisites/playbook.yml b/k8s/prerequisites/playbook.yml deleted file mode 100644 index 9eec5c17..00000000 --- a/k8s/prerequisites/playbook.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Kuberneteslochy - hosts: kubernetes-lochy-k8s - roles: - # No mounts are configured, this just installs required packages - - role: ome.nfs_mount diff --git a/molecule/bootstrap/molecule.yml b/molecule/bootstrap/molecule.yml deleted file mode 100644 index 359f74fe..00000000 --- a/molecule/bootstrap/molecule.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -dependency: - name: galaxy - options: - role-file: requirements.yml -driver: - name: docker -lint: | - yamllint . - ansible-lint - flake8 -platforms: - - name: omedev - image: centos:7 -provisioner: - name: ansible - playbooks: - converge: ../../bootstrap/playbook.yml - lint: - name: ansible-lint -scenario: - name: bootstrap - test_sequence: - - lint - - dependency - - syntax -verifier: - name: testinfra diff --git a/molecule/docker-prod/Dockerfile.j2 b/molecule/docker-prod/Dockerfile.j2 deleted file mode 100644 index 7e2d467d..00000000 --- a/molecule/docker-prod/Dockerfile.j2 +++ /dev/null @@ -1,22 +0,0 @@ -# Molecule managed - -{% if item.registry is defined %} -FROM {{ item.registry.url }}/{{ item.image }} -{% else %} -FROM {{ item.image }} -{% endif %} - -{% if item.env is defined %} -{% for var, value in item.env.items() %} -{% if value %} -ENV {{ var }} {{ value }} -{% endif %} -{% endfor %} -{% endif %} - -RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo python-jmespath bash ca-certificates iproute2 && apt-get clean; \ - elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash iproute && dnf clean all; \ - elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo python36-jmespath python38-jmespath yum-plugin-ovl bash iproute ca-certificates && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ - elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml iproute2 && zypper clean -a; \ - elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ - elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates iproute2 && xbps-remove -O; fi diff --git a/molecule/docker-prod/converge.yml b/molecule/docker-prod/converge.yml deleted file mode 100644 index 775d3456..00000000 --- a/molecule/docker-prod/converge.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -- name: Converge - hosts: ome-dockr-prod1.openmicroscopy.org - tasks: - - name: Workaround to get host IP inside docker - shell: hostname -I | cut -d' ' -f1 - register: hostname_ip - check_mode: false - changed_when: false - tags: - # Ignore [306] Shells that use pipes should set the pipefail option - - skip_ansible_lint - - - name: Set address of postgres for redmine - set_fact: - redmine_tracker_db_host: "{{ hostname_ip.stdout }}" - -- name: Import-playbook - import_playbook: ../../omedev/docker-prod-apps.yml diff --git a/molecule/docker-prod/molecule.yml b/molecule/docker-prod/molecule.yml deleted file mode 100644 index f43e3aa3..00000000 --- a/molecule/docker-prod/molecule.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- -dependency: - name: galaxy - options: - role-file: requirements.yml -driver: - name: docker -lint: | - yamllint . - ansible-lint - flake8 -platforms: - - name: ome-dockr-prod1.openmicroscopy.org - image: centos/systemd:latest - command: /sbin/init - privileged: true - groups: - - docker-hosts - - omedev-docker - published_ports: - - "0.0.0.0:9090:9090/tcp" -provisioner: - name: ansible - playbooks: - prepare: prepare.yml - converge: converge.yml - inventory: - host_vars: - ome-dockr-prod1.openmicroscopy.org: - ome_monitored_node_exporter_hosts: - - node.example.org - ome_monitored_postgres_hosts: - - pg.example.org - ome_monitored_omero_server_hosts: - - omeroserver.example.org - ome_monitored_omero_web_hosts: - - omeroweb.example.org - prometheus_docker_data_volume: /srv/prometheus - nfs_minio_data_volume: /srv/minio - redmine_tracker_docker_data_volume: /srv/redmine-files - - group_vars: - # all: - # molecule_test: true - docker-hosts: - # This should allow docker-in-docker to work - docker_storage_driver: vfs - # Latest version 17.12.1.ce-1.el7.centos has a bug that prevents - # testing on travis: https://github.com/docker/for-linux/issues/219 - docker_version: 17.09.1.ce-1.el7.centos - lint: - name: ansible-lint -scenario: - name: docker-prod -verifier: - name: testinfra diff --git a/molecule/docker-prod/prepare.yml b/molecule/docker-prod/prepare.yml deleted file mode 100644 index 7bb8ba41..00000000 --- a/molecule/docker-prod/prepare.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- name: Prepare - hosts: ome-dockr-prod1.openmicroscopy.org - roles: - - role: ome.postgresql - postgresql_version: "13" - postgresql_server_auth: - - database: redmine - user: redmine - address: 0.0.0.0/0 - postgresql_databases: - - name: redmine - owner: redmine - postgresql_users: - - user: redmine - password: redmine - databases: - - redmine - postgresql_server_listen: "'*'" - -- name: Import playbook - import_playbook: ../../omedev/playbook.yml diff --git a/molecule/docker-prod/tests/test_default.py b/molecule/docker-prod/tests/test_default.py deleted file mode 100644 index ced9ed28..00000000 --- a/molecule/docker-prod/tests/test_default.py +++ /dev/null @@ -1,39 +0,0 @@ -import json -import os -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -def test_prometheus_targets(host): - out = host.check_output( - 'curl -k -f --user admin:monitoring ' - 'https://localhost/prometheus/api/v1/targets') - d = json.loads(out) - assert d['status'] == 'success' - assert d['data']['droppedTargets'] == [] - unique_instances = set( - t['labels']['instance'] for t in d['data']['activeTargets']) - assert len({ - 'node.example.org:443', - 'pg.example.org:443', - 'omeroserver.example.org:443', - 'omeroweb.example.org:443', - 'idr.openmicroscopy.org:443', - 'idr1.openmicroscopy.org:443', - 'idr2.openmicroscopy.org:443', - 'localhost:9090', - }.difference(unique_instances)) == 0 - - -def test_minio_connect(host): - out = host.check_output('curl -s http://localhost:9000 -I') - assert 'Server: MinIO/' in out - - -def test_redmine_connect(host): - out = host.check_output( - 'curl -k -f -L -H "Host: idr-redmine-docker.openmicroscopy.org" ' - 'https://localhost/') - assert 'Redmine' in out diff --git a/molecule/nightshade-webclients/molecule.yml b/molecule/nightshade-webclients/molecule.yml deleted file mode 100644 index 71cd0c18..00000000 --- a/molecule/nightshade-webclients/molecule.yml +++ /dev/null @@ -1,58 +0,0 @@ ---- -dependency: - name: galaxy - options: - role-file: requirements.yml -driver: - name: docker -lint: | - yamllint . - ansible-lint - flake8 -platforms: - - name: ns-webclients - image: centos/systemd - command: /sbin/init - privileged: true - groups: - - docker-hosts - - omero-web - - monitored -provisioner: - name: ansible - options: - diff: true - inventory: - group_vars: - all: - molecule_test: true - docker-hosts: - # firewalld isn't installed, don't attempt to disable - iptables_raw_disable_firewalld: false - playbooks: - converge: ../../site.yml - lint: - name: ansible-lint -scenario: - name: nightshade-webclients - converge_sequence: - - converge - test_sequence: - - destroy - # dependency must come first so that ansible-lint will see a custom module - # This might be fixed by https://github.com/ansible/molecule/pull/1739 - - dependency - - lint - - syntax - - create - - prepare - - converge - # FIXME: Some tasks are not idempotent - # - idempotence - ################################################################################ - # FIXME: Tests hang on Travis but pass locally - # - verify - ################################################################################ - - destroy -verifier: - name: testinfra diff --git a/molecule/nightshade-webclients/tests/test_default.py b/molecule/nightshade-webclients/tests/test_default.py deleted file mode 100644 index a88b0cc6..00000000 --- a/molecule/nightshade-webclients/tests/test_default.py +++ /dev/null @@ -1,40 +0,0 @@ -import os -import pytest -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - -OMERO = '/opt/omero/server/OMERO.server/bin/omero' -OMERO_LOGIN = '-C -s localhost -u root -w omero' - - -@pytest.mark.parametrize("name", [ - 'nginx', - 'omero-web', - 'prometheus-node-exporter', -]) -def test_service_running_and_enabled(host, name): - service = host.service(name) - assert service.is_running - assert service.is_enabled - - -def test_omero_metrics(host): - out = host.check_output( - 'curl -f -u monitoring:monitoring -k ' - 'https://localhost/django_prometheus/metrics') - assert "django_http_responses_body_total_bytes_count" in out - - -def test_omero_metrics_auth_fail(host): - out = host.run( - 'curl -f -u monitoring:incorrect -k ' - 'https://localhost/django_prometheus/metrics') - assert out.rc == 22 - assert '401' in out.stderr - - -def test_omero_nginx_ssl(host): - out = host.check_output('curl -fkI https://localhost/') - assert 'Location: /webclient/' in out diff --git a/molecule/ome-demoserver/molecule.yml b/molecule/ome-demoserver/molecule.yml index 3e399874..b09507f8 100644 --- a/molecule/ome-demoserver/molecule.yml +++ b/molecule/ome-demoserver/molecule.yml @@ -11,23 +11,53 @@ lint: | flake8 platforms: - name: ome-demoserver - image: centos:7 + image: eniocarboni/docker-rockylinux-systemd:9 + image_version: latest + command: /sbin/init + privileged: true + cgroupns_mode: host + tmpfs: + - /sys/fs/cgroup groups: + - docker-hosts + - omero-py3 - ome-demoservers provisioner: name: ansible playbooks: - converge: ../../site.yml + prepare: ../prepare.yml + converge: ../../omero/ome-demoserver.yml + options: + v: true + diff: true + inventory: + host_vars: + ome-demoserver: + postgresql_version: "16" + omero_server_selfsigned_certificates: true + omero_server_dbhost: localhost + omero_server_dbuser: demo + omero_server_dbname: omero + omero_server_dbpassword: omero + omero_server_mail_from: example.com + omero_server_mail_host: localhost + omero_server_rootpassword: omero lint: name: ansible-lint scenario: name: ome-demoserver test_sequence: + # dependency must come first, otherwise the first "destroy" errors + - dependency - destroy - # dependency must come first so that ansible-lint will see a custom module - # This might be fixed by https://github.com/ansible/molecule/pull/1739 - dependency - - lint - syntax + - create + - prepare + - converge + # FIXME: Some tasks are not idempotent + # - idempotence + - verify + - destroy verifier: name: testinfra diff --git a/molecule/ome-dundeeomero/Dockerfile.j2 b/molecule/ome-dundeeomero/Dockerfile.j2 deleted file mode 100644 index 00b7fd61..00000000 --- a/molecule/ome-dundeeomero/Dockerfile.j2 +++ /dev/null @@ -1,22 +0,0 @@ -# Molecule managed - -{% if item.registry is defined %} -FROM {{ item.registry.url }}/{{ item.image }} -{% else %} -FROM {{ item.image }} -{% endif %} - -{% if item.env is defined %} -{% for var, value in item.env.items() %} -{% if value %} -ENV {{ var }} {{ value }} -{% endif %} -{% endfor %} -{% endif %} - -RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates iproute2 && apt-get clean; \ - elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash iproute && dnf clean all; \ - elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash iproute ca-certificates && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ - elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml iproute2 && zypper clean -a; \ - elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ - elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates iproute2 && xbps-remove -O; fi diff --git a/molecule/ome-dundeeomero/molecule.yml b/molecule/ome-dundeeomero/molecule.yml deleted file mode 100644 index 8260f968..00000000 --- a/molecule/ome-dundeeomero/molecule.yml +++ /dev/null @@ -1,59 +0,0 @@ ---- -dependency: - name: galaxy - options: - role-file: requirements.yml -driver: - name: docker -lint: | - yamllint . - ansible-lint - flake8 -platforms: - - name: ome-dundeeomero.openmicroscopy.org - image: centos/systemd - command: /sbin/init - privileged: true - groups: - - docker-hosts - - omero-server - - monitored -provisioner: - name: ansible - options: - diff: true - # skip-tags: - # - "skip_molecule" - inventory: - group_vars: - all: - molecule_test: true - docker-hosts: - # firewalld isn't installed, don't attempt to disable - iptables_raw_disable_firewalld: false - playbooks: - converge: ../../site.yml - lint: - name: ansible-lint - # env: - # ANSIBLE_ROLES_PATH: ../../vendor -scenario: - name: ome-dundeeomero - converge_sequence: - - converge - test_sequence: - - destroy - # dependency must come first so that ansible-lint will see a custom module - # This might be fixed by https://github.com/ansible/molecule/pull/1739 - - dependency - - lint - - syntax - - create - - prepare - - converge - # FIXME: Some tasks are not idempotent - # - idempotence - - verify - - destroy -verifier: - name: testinfra diff --git a/molecule/ome-dundeeomero/tests/test_default.py b/molecule/ome-dundeeomero/tests/test_default.py deleted file mode 100644 index dc1ac485..00000000 --- a/molecule/ome-dundeeomero/tests/test_default.py +++ /dev/null @@ -1,46 +0,0 @@ -import os -import pytest -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - -OMERO = '/opt/omero/server/OMERO.server/bin/omero' -OMERO_LOGIN = '-C -s localhost -u root -w omero' - - -@pytest.mark.parametrize("name", [ - 'nginx', - 'omero-server', - 'postgresql-11', - 'prometheus-node-exporter', - 'prometheus-omero-exporter', - 'prometheus-postgres-exporter', -]) -def test_service_running_and_enabled(host, name): - service = host.service(name) - assert service.is_running - assert service.is_enabled - - -def test_omero_login(host): - with host.sudo('omero-server'): - host.check_output( - '/opt/omero/server/OMERO.server/bin/omero ' - 'login -C -s localhost -u root -w omero') - - -@pytest.mark.parametrize("curl", [ - 'localhost:9449/metrics', - '-u monitoring:monitoring -k https://localhost/metrics/9449', -]) -def test_omero_metrics(host, curl): - out = host.check_output('curl -f %s' % curl) - assert 'omero_sessions_active' in out - - -def test_omero_metrics_auth_fail(host): - out = host.run( - 'curl -f -u monitoring:incorrect -k https://localhost/metrics/9449') - assert out.rc == 22 - assert '401' in out.stderr diff --git a/molecule/ome-pg-prod/molecule_disabled.yml b/molecule/ome-pg-prod/molecule_disabled.yml deleted file mode 100644 index b2feddfb..00000000 --- a/molecule/ome-pg-prod/molecule_disabled.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -dependency: - name: galaxy - options: - role-file: requirements.yml -driver: - name: docker -lint: | - yamllint . - ansible-lint - flake8 -platforms: - - name: ome-pg-prod1.openmicroscopy.org - image: centos/systemd - image_version: latest - command: /sbin/init - privileged: true -provisioner: - name: ansible - playbooks: - prepare: prepare.yml - converge: ../../postgres/ome-pg-prod.yml - lint: - name: ansible-lint -scenario: - name: ome-pg-prod -verifier: - name: testinfra diff --git a/molecule/ome-pg-prod/prepare.yml b/molecule/ome-pg-prod/prepare.yml deleted file mode 100644 index 04a5e83f..00000000 --- a/molecule/ome-pg-prod/prepare.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Workaround lack of cron on Docker -- name: Prepare pg prod - hosts: ome-pg-prod1.openmicroscopy.org - tasks: - - name: Install cron - become: true - ansible.builtin.yum: - name: cronie - state: present diff --git a/molecule/ome-pg-prod/tests/test_default.py b/molecule/ome-pg-prod/tests/test_default.py deleted file mode 100644 index baa73f3b..00000000 --- a/molecule/ome-pg-prod/tests/test_default.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -def test_service_running_and_enabled(host): - assert host.service('postgresql-9.6').is_running - assert host.service('postgresql-9.6').is_enabled - - -def test_dbs(host): - out = host.check_output( - 'PGPASSWORD=idr-redmine psql -hlocalhost -Uidr-redmine -l -tA') - assert 'idr-redmine|idr-redmine|UTF8|' in out diff --git a/molecule/omero-training-server/Dockerfile.j2 b/molecule/omero-training-server/Dockerfile.j2 deleted file mode 100644 index 00b7fd61..00000000 --- a/molecule/omero-training-server/Dockerfile.j2 +++ /dev/null @@ -1,22 +0,0 @@ -# Molecule managed - -{% if item.registry is defined %} -FROM {{ item.registry.url }}/{{ item.image }} -{% else %} -FROM {{ item.image }} -{% endif %} - -{% if item.env is defined %} -{% for var, value in item.env.items() %} -{% if value %} -ENV {{ var }} {{ value }} -{% endif %} -{% endfor %} -{% endif %} - -RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates iproute2 && apt-get clean; \ - elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash iproute && dnf clean all; \ - elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash iproute ca-certificates && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ - elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml iproute2 && zypper clean -a; \ - elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ - elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates iproute2 && xbps-remove -O; fi diff --git a/molecule/omero-training-server/molecule.yml b/molecule/omero-training-server/molecule.yml deleted file mode 100644 index af145c55..00000000 --- a/molecule/omero-training-server/molecule.yml +++ /dev/null @@ -1,59 +0,0 @@ ---- -dependency: - name: galaxy - options: - role-file: requirements.yml -driver: - name: docker -lint: | - yamllint . - ansible-lint - flake8 -platforms: - - name: ome-outreach - image: centos/systemd - command: /sbin/init - privileged: true - groups: - - docker-hosts - - omero-server - - omero-web - - monitored -provisioner: - name: ansible - inventory: - group_vars: - all: - molecule_test: true - postgresql_version: "13" - docker-hosts: - # This should allow docker-in-docker to work - docker_storage_driver: vfs - # Latest version 17.12.1.ce-1.el7.centos has a bug that prevents - # testing on travis: https://github.com/docker/for-linux/issues/219 - docker_version: 17.09.1.ce-1.el7.centos - # firewalld isn't installed, don't attempt to disable - iptables_raw_disable_firewalld: false - playbooks: - prepare: ../resources/prepare-iproute.yml - converge: ../../site.yml - lint: - name: ansible-lint -scenario: - name: omero-training-server - test_sequence: - - destroy - # dependency must come first so that ansible-lint will see a custom module - # This might be fixed by https://github.com/ansible/molecule/pull/1739 - - dependency - - lint - - syntax - - create - - prepare - - converge - # FIXME: Some tasks are not idempotent - # - idempotence - - verify - - destroy -verifier: - name: testinfra diff --git a/molecule/omero-training-server/tests/test_default.py b/molecule/omero-training-server/tests/test_default.py deleted file mode 100644 index 1660aff8..00000000 --- a/molecule/omero-training-server/tests/test_default.py +++ /dev/null @@ -1,63 +0,0 @@ -import os -import pytest -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - -OMERO = '/opt/omero/server/OMERO.server/bin/omero' -OMERO_LOGIN = '-C -s localhost -u root -w omero' - - -@pytest.mark.parametrize("name", [ - 'nginx', - 'omero-server', - 'omero-web', - 'postgresql-13', - 'prometheus-node-exporter', - 'prometheus-omero-exporter', - 'prometheus-postgres-exporter', -]) -def test_service_running_and_enabled(host, name): - service = host.service(name) - assert service.is_running - assert service.is_enabled - - -def test_omero_login(host): - with host.sudo('importer1'): - host.check_output( - '/opt/omero/server/OMERO.server/bin/omero ' - 'login -C -s localhost -u root -w omero') - - -@pytest.mark.parametrize("curl", [ - 'localhost:9449/metrics', - '-u monitoring:monitoring -k https://localhost/metrics/9449', -]) -def test_omero_metrics(host, curl): - out = host.check_output('curl -f %s' % curl) - assert 'omero_sessions_active' in out - - -def test_omero_metrics_auth_fail(host): - out = host.run( - 'curl -f -u monitoring:incorrect -k https://localhost/metrics/9449') - assert out.rc == 22 - assert '401' in out.stderr - - -def test_omero_nginx_ssl(host): - out = host.check_output('curl -fkI https://localhost/') - assert 'Location: /webclient/' in out - - -def test_local_ldap(host): - initialised = host.check_output( - '/home/ldap/ldapmanager get dc=openmicroscopy,dc=org') - if len(initialised.strip()) == 0: - host.check_output('/home/ldap/ldapmanager init') - - out = host.check_output( - '/home/ldap/ldapmanager get dc=openmicroscopy,dc=org') - assert 'dn: dc=openmicroscopy,dc=org' in out diff --git a/molecule/prepare.yml b/molecule/prepare.yml new file mode 100644 index 00000000..97571d71 --- /dev/null +++ b/molecule/prepare.yml @@ -0,0 +1,12 @@ +--- +- name: Converge + hosts: all + + # If testing in Docker cron won't be installed + pre_tasks: + - name: Install cron + become: true + ansible.builtin.dnf: + update_cache: true + name: cronie + state: present diff --git a/molecule/release/molecule.yml b/molecule/release/molecule.yml deleted file mode 100644 index add768eb..00000000 --- a/molecule/release/molecule.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -dependency: - name: galaxy -driver: - name: docker -lint: | - yamllint . - ansible-lint - flake8 -platforms: - - name: release - image: centos:7 - groups: - - idr0-slot3.openmicroscopy.org - - name: prerelease - image: centos:7 - groups: - - idr0-slot3.openmicroscopy.org -provisioner: - name: ansible - playbooks: - converge: ../../release/release-acceptance.yml - inventory: - group_vars: - idr0-slot3.openmicroscopy.org: - product: component - host_vars: - prerelease: - version: '3.2.0-rc1' - release: - version: 3.2.0 - lint: - name: ansible-lint -scenario: - name: release -verifier: - name: testinfra diff --git a/molecule/release/prepare.yml b/molecule/release/prepare.yml deleted file mode 100644 index 7f4743fd..00000000 --- a/molecule/release/prepare.yml +++ /dev/null @@ -1,72 +0,0 @@ ---- -- name: Prepare release - hosts: all - vars: - www_folders: - - /uod/idr/www/docs.openmicroscopy.org - - /uod/idr/www/downloads.openmicroscopy.org - releases: - - 3.2.0 - - 3.2.0-rc1 - tasks: - - name: Create existing released components - ansible.builtin.file: - path: "{{ item }}/component/3.1.8" - state: directory - owner: root - group: root - mode: 01555 - with_items: "{{ www_folders }}" - - name: Create minor version directory - ansible.builtin.file: - path: "{{ item }}/component/3.1" - state: directory - with_items: "{{ www_folders }}" - - name: Create minor version redirects - ansible.builtin.copy: - dest: "{{ item }}/component/3.1/.htaccess" - content: "Redirect 301 /component/3.1 /component/3.1.8" - with_items: "{{ www_folders }}" - - name: Create major version directory - ansible.builtin.file: - path: "{{ item }}/component/3" - state: directory - with_items: "{{ www_folders }}" - - name: Create major version redirects - ansible.builtin.copy: - dest: "{{ item }}/component/3/.htaccess" - content: "Redirect 301 /component/3 /component/3.1.8" - with_items: "{{ www_folders }}" - - name: Create latest version directory - ansible.builtin.file: - path: "{{ item }}/component/latest" - state: directory - with_items: "{{ www_folders }}" - - name: Create latest version redirects - ansible.builtin.copy: - dest: "{{ item }}/component/latest/.htaccess" - content: "Redirect 301 /component/latest /component/3.1.8" - with_items: "{{ www_folders }}" - - name: Create new release components - ansible.builtin.file: - path: "{{ item[0] }}/component/{{ item[1] }}" - state: directory - mode: 01777 - with_nested: - - "{{ www_folders }}" - - "{{ releases }}" - - name: Create .htaccess file - ansible.builtin.file: - path: "{{ item[0] }}/component/{{ item[1] }}/.htaccess" - state: touch - with_nested: - - "{{ www_folders }}" - - "{{ releases }}" - - name: Create mock content - ansible.builtin.file: - path: "{{ item[0] }}/component/{{ item[1] }}/test" - state: touch - mode: 01777 - with_nested: - - "{{ www_folders }}" - - "{{ releases }}" diff --git a/molecule/release/tests/test_default.py b/molecule/release/tests/test_default.py deleted file mode 100644 index b1961472..00000000 --- a/molecule/release/tests/test_default.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -import pytest -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - -DOWNLOADS_URL = "/uod/idr/www/downloads.openmicroscopy.org" -DOCS_URL = "/uod/idr/www/docs.openmicroscopy.org" - - -@pytest.mark.parametrize('base_folder', [DOWNLOADS_URL, DOCS_URL]) -def test_permissions(host, base_folder): - v = host.ansible.get_variables() - assert not host.file('%s/component/%s/.htaccess' % ( - base_folder, v['version'])).exists - - f = host.file('%s/component/%s' % (base_folder, v['version'])) - assert f.exists - assert f.user == 'root' - assert oct(f.mode) == '0o1555' - - -@pytest.mark.parametrize('base_folder', [DOWNLOADS_URL, DOCS_URL]) -def test_redirects(host, base_folder): - v = host.ansible.get_variables() - hostname = host.backend.get_hostname() - f = host.file('%s/component/3.2/.htaccess' % base_folder) - if hostname == 'release': - assert f.exists - assert f.content_string == ( - 'Redirect 301 /component/3.2 /component/%s' % v['version']) - elif hostname == 'prelease': - assert not f.exists - f = host.file('%s/component/3/.htaccess' % base_folder) - assert f.exists - if hostname == 'release': - assert f.content_string == ( - 'Redirect 301 /component/3 /component/%s' % v['version']) - elif hostname == 'prelease': - assert f.content_string == 'Redirect 301 /component/3 /component/3.1.8' - f = host.file('%s/component/latest/.htaccess' % base_folder) - assert f.exists - if hostname == 'release': - assert f.content_string == ( - 'Redirect 301 /component/latest /component/%s' % v['version']) - elif hostname == 'prelease': - assert (f.content_string == - 'Redirect 301 /component/latest /component/3.1.8') diff --git a/molecule/resources/Dockerfile.j2 b/molecule/resources/Dockerfile.j2 deleted file mode 100644 index 00b7fd61..00000000 --- a/molecule/resources/Dockerfile.j2 +++ /dev/null @@ -1,22 +0,0 @@ -# Molecule managed - -{% if item.registry is defined %} -FROM {{ item.registry.url }}/{{ item.image }} -{% else %} -FROM {{ item.image }} -{% endif %} - -{% if item.env is defined %} -{% for var, value in item.env.items() %} -{% if value %} -ENV {{ var }} {{ value }} -{% endif %} -{% endfor %} -{% endif %} - -RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates iproute2 && apt-get clean; \ - elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash iproute && dnf clean all; \ - elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash iproute ca-certificates && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ - elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml iproute2 && zypper clean -a; \ - elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ - elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates iproute2 && xbps-remove -O; fi diff --git a/molecule/resources/prepare-iproute.yml b/molecule/resources/prepare-iproute.yml deleted file mode 100644 index 1730ad08..00000000 --- a/molecule/resources/prepare-iproute.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Install iproute for ansible network vars -- name: Prepare iproute in resources - hosts: all - tasks: - - name: Install iproute - become: true - ansible.builtin.yum: - name: iproute - state: present - - - name: Install cron - become: true - ansible.builtin.yum: - name: cronie - state: present diff --git a/molecule/web-proxy/molecule.yml b/molecule/web-proxy/molecule.yml deleted file mode 100644 index 8bdf86d9..00000000 --- a/molecule/web-proxy/molecule.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- -dependency: - name: galaxy - options: - role-file: requirements.yml -driver: - name: docker -lint: | - yamllint . - ansible-lint - flake8 -platforms: - - name: web-proxy - image: centos:7 - groups: - - web-proxies -provisioner: - name: ansible - playbooks: - converge: ../../web-proxy/playbook.yml - lint: - name: ansible-lint -scenario: - name: web-proxy - test_sequence: - - lint - - dependency - - syntax -verifier: - name: testinfra diff --git a/molecule/www/Dockerfile.j2 b/molecule/www/Dockerfile.j2 deleted file mode 120000 index 0e9184b4..00000000 --- a/molecule/www/Dockerfile.j2 +++ /dev/null @@ -1 +0,0 @@ -../resources/Dockerfile.j2 \ No newline at end of file diff --git a/molecule/www/molecule.yml b/molecule/www/molecule.yml deleted file mode 100644 index e6b232df..00000000 --- a/molecule/www/molecule.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -dependency: - name: galaxy - options: - role-file: requirements.yml -driver: - name: docker -lint: | - yamllint . - ansible-lint - flake8 -platforms: - - name: www - image: centos/systemd - image_version: latest - command: /sbin/init - privileged: true -provisioner: - name: ansible - playbooks: - converge: ../../www/www-deploy.yml - lint: - name: ansible-lint -scenario: - name: www -verifier: - name: testinfra diff --git a/molecule/www/tests/test_default.py b/molecule/www/tests/test_default.py deleted file mode 100644 index d61e82f1..00000000 --- a/molecule/www/tests/test_default.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import pytest -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - - -@pytest.mark.skip(reason="Causes Travis CI to exceed 10min timeout") -@pytest.mark.parametrize("address", [ - "http://localhost/", - "https://localhost/", -]) -def test_web(host, address): - out = host.check_output('curl -k %s' % address) - assert 'Home | Open Microscopy Environment (OME)' in out - - -@pytest.mark.skip(reason="Causes Travis CI to exceed 10min timeout") -def test_archived_community(host): - out = host.check_output('curl -kL https://localhost/community') - assert 'Powered by phpBB' in out - - out = host.check_output('curl -kIL https://localhost/community') - assert 'Set-Cookie: phpbb' not in out diff --git a/molecule/www/tests/test_redirects.py b/molecule/www/tests/test_redirects.py deleted file mode 100644 index 3dfa70ac..00000000 --- a/molecule/www/tests/test_redirects.py +++ /dev/null @@ -1,116 +0,0 @@ -import os -import testinfra.utils.ansible_runner -import pytest - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') - -external_uris = [ - ('/forums', 'https://forum.image.sc/c/data-management'), - ('/omero-blog', 'http://blog.openmicroscopy.org'), - ('/site/about/development-teams/glencoe-software', - 'https://www.glencoesoftware.com/team.html'), - ('/site/community/scripts', - 'https://docs.openmicroscopy.org/latest/omero/developers/' - 'scripts/index.html'), - ('/site/support/bio-formats', - 'https://docs.openmicroscopy.org/latest/bio-formats/'), - ('/site/support/omero', - 'https://docs.openmicroscopy.org/latest/omero/'), - ('/site/support/ome-model', - 'https://docs.openmicroscopy.org/latest/ome-model/'), - ('/site/support/file-formats', - 'https://docs.openmicroscopy.org/latest/ome-model/'), - ('/site/support/file-formats/schemas/specifications/' - 'compliant-file-specification', - 'https://docs.openmicroscopy.org/latest/ome-model/specifications/'), - ('/site/support/ome-tiff', - 'https://docs.openmicroscopy.org/latest/ome-model/ome-tiff/'), - ('/site/support/ome-files-cpp', - 'https://docs.openmicroscopy.org/latest/ome-files-cpp/'), - ('/site/support/contributing', - 'https://docs.openmicroscopy.org/contributing/'), - ('/info/flimfit', 'http://flimfit.org'), - ('/info/scripts', - 'https://docs.openmicroscopy.org/latest/omero/developers/' - 'scripts/index.html'), - ('/info/bio-formats', - 'https://docs.openmicroscopy.org/latest/bio-formats/'), - ('/info/slidebook', - 'https://www.intelligent-imaging.com/technical-answers'), -] - -redirect_uris = [ - ('/site', '/'), - ('/site/about', '/about'), - ('/site/about/licensing', '/licensing'), - ('/site/about/licensing-attribution', '/licensing'), - ('/site/about/licensing-attribution/licensing', '/licensing'), - ('/site/about/ome-contributors', '/contributors'), - ('/site/about/partners', '/commercial-partners'), - ('/site/about/development-teams', '/teams'), - ('/site/about/publications', '/citing-ome'), - ('/site/about/who-ome', '/teams'), - ('/site/about/what-omero/overview', '/omero'), - ('/site/about/roadmap', '/about'), - ('/site/about/project-history', '/about'), - - ('/site/community', '/support'), - ('/site/community/mailing-lists', '/support'), - ('/site/events', '/events'), - ('/site/community/minutes/conference-calls', '/on-the-web'), - ('/site/community/minutes/meetings/12th-annual-users-meeting-2017', - '/events/12th-annual-users-meeting-2017.html'), - ('/site/community/minutes/meetings/11th-annual-users-meeting-2016', - '/events/11th-annual-users-meeting-2016.html'), - ('/site/community/minutes/meetings/10th-annual-users-meeting-june-2015', - '/events/10th-annual-users-meeting-june-2015.html'), - ('/site/community/minutes/meetings/9th-annual-users-meeting-june-2014', - '/events/9th-annual-users-meeting-june-2014.html'), - ('/site/community/jobs', '/careers'), - - ('/site/products', '/products'), - ('/site/products/bio-formats', '/bio-formats'), - ('/site/products/bio-formats/downloads', '/bio-formats/downloads/'), - ('/site/products/omero', '/omero'), - ('/site/products/omero/downloads', '/omero/downloads/'), - ('/site/products/omero/feature-list', '/omero/features/'), - ('/site/products/omero/secvuln', '/security/advisories/'), - ('/site/products/ome5/secvuln', '/security/advisories/'), - ('/site/products/omero/secvuln/2014-SV3-csrf', - '/security/advisories/2014-SV3-csrf/'), - - ('/site/support', '/docs'), - ('/site/support/ome-artwork', '/artwork'), - ('/site/support/ome-artwork/artwork-usage', '/artwork'), - ('/site/news', '/announcements'), - - ('/info/vulnerabilities', '/security/advisories/'), - ('/info/vulnerabilities/2014-SV3-csrf', - '/security/advisories/2014-SV3-csrf/'), - ('/info/omero', '/omero'), - ('/info/cls', '/omero/downloads/'), - ('/info/download', '/omero/downloads/'), - ('/info/downloads', '/omero/downloads/'), - ('/info/attribution', '/licensing/'), -] - - -@pytest.mark.skip(reason="Causes Travis CI to exceed 10min timeout") -@pytest.mark.parametrize('path,redirect', redirect_uris) -def test_internal_redirects(host, path, redirect): - out = host.check_output('curl -I http://localhost%s' % path) - assert 'HTTP/1.1 302' in out - assert 'Location: http://localhost%s' % redirect in out - - -@pytest.mark.skip(reason="Causes Travis CI to exceed 10min timeout") -@pytest.mark.parametrize('path,redirect', external_uris) -def test_external_redirects(host, path, redirect): - out = host.check_output('curl -I http://localhost%s' % path) - assert 'HTTP/1.1 302' in out - assert 'Location: %s' % redirect in out - - out = host.check_output('curl -I http://localhost%s/' % path) - assert 'HTTP/1.1 302' in out - assert 'Location: %s' % redirect in out diff --git a/omedev/docker-prod-apps.yml b/omedev/docker-prod-apps.yml deleted file mode 100644 index f47dbf6e..00000000 --- a/omedev/docker-prod-apps.yml +++ /dev/null @@ -1,349 +0,0 @@ ---- -# Production Docker applications - -# May want to change this to a hostgroup -- hosts: ome-dockr-prod1.openmicroscopy.org - - pre_tasks: - - - name: Create Prometheus data directory - become: true - file: - path: "{{ prometheus_docker_data_volume }}" - owner: "{{ prometheus_docker_user }}" - group: root - state: directory - mode: 0755 - - - name: Create Minio data directory - become: true - file: - path: "{{ nfs_minio_data_volume }}" - owner: root - group: root - state: directory - mode: 0755 - - - name: Create Redmine data directory - become: true - file: - path: "{{ redmine_tracker_docker_data_volume }}" - owner: "{{ redmine_docker_user }}" - group: "{{ redmine_docker_user }}" - state: directory - mode: 0755 - - roles: - - # Self monitoring - - role: ome.prometheus_node - - - role: ome.prometheus - prometheus_docker_network: monitoring - # https://prometheus.io/docs/guides/basic-auth/#prometheus-configuration - prometheus_additional_command_args: >- - --storage.tsdb.retention.time=5y - --web.external-url https://$(hostname)/prometheus - --web.route-prefix=/ - - # prometheus_alertmanager_slack_webhook: "secret" - # prometheus_alertmanager_slack_channel: "#secret" - - # Disable external ports, external access is via a proxy - prometheus_port: 0 - prometheus_alertmanager_port: 0 - prometheus_blackboxexporter_port: 0 - - prometheus_targets: - - jobname: self-node-exporter - hosts: - - "{{ ansible_default_ipv4.address }}" - port: 9100 - - jobname: self-prometheus - hosts: - - localhost - port: 9090 - - prometheus_custom_targets: - - - job_name: node-exporter - basic_auth: - username: "{{ scrape_username }}" - password: "{{ scrape_password }}" - metrics_path: /metrics/9100 - scheme: https - static_configs: - - targets: "{{ monitored_node_exporter_hosts }}" - - - job_name: omero-web - basic_auth: - username: "{{ scrape_username }}" - password: "{{ scrape_password }}" - metrics_path: /django_prometheus/metrics - scheme: https - static_configs: - - targets: "{{ monitored_omero_web_hosts }}" - - - job_name: jmx-blitz - basic_auth: - username: "{{ scrape_username }}" - password: "{{ scrape_password }}" - metrics_path: /metrics/9180 - scheme: https - static_configs: - - targets: "{{ monitored_omero_server_hosts }}" - - - job_name: jmx-indexer - basic_auth: - username: "{{ scrape_username }}" - password: "{{ scrape_password }}" - metrics_path: /metrics/9181 - scheme: https - static_configs: - - targets: "{{ monitored_omero_server_hosts }}" - - - job_name: jmx-pixeldata - basic_auth: - username: "{{ scrape_username }}" - password: "{{ scrape_password }}" - metrics_path: /metrics/9182 - scheme: https - static_configs: - - targets: "{{ monitored_omero_server_hosts }}" - - - job_name: postgres-exporter - basic_auth: - username: "{{ scrape_username }}" - password: "{{ scrape_password }}" - metrics_path: /metrics/9187 - scheme: https - static_configs: - - targets: "{{ monitored_postgres_hosts }}" - - - job_name: omero-server - basic_auth: - username: "{{ scrape_username }}" - password: "{{ scrape_password }}" - metrics_path: /metrics/9449 - scheme: https - static_configs: - - targets: "{{ monitored_omero_server_hosts }}" - -# Federated metrics from the IDR -# To get all federated metrics so you can decide what to fetch: -# curl -G https://idr.openmicroscopy.org/prometheus/federate --data-urlencode 'match[]={__name__=~".+"}' - - - job_name: federate-{{ idr_internal_1 }} - honor_labels: true - metrics_path: /prometheus/federate - params: - 'match[]': - # TODO: Decide what metrics to fetch - - '{job="node-exporter"}' - - '{job="omero-server"}' - scheme: https - static_configs: - - targets: - - "{{ idr_internal_1 }}.openmicroscopy.org" - labels: - prometheussrc: "{{ idr_internal_1 }}" - - - job_name: federate-{{ idr_internal_2 }} - honor_labels: true - metrics_path: /prometheus/federate - params: - 'match[]': - # TODO: Decide what metrics to fetch - - '{job="node-exporter"}' - - '{job="omero-server"}' - scheme: https - static_configs: - - targets: - - "{{ idr_internal_2 }}.openmicroscopy.org" - labels: - prometheussrc: "{{ idr_internal_2 }}" - - - job_name: federate-idr-production - honor_labels: true - metrics_path: /prometheus/federate - params: - 'match[]': - # TODO: Decide what metrics to fetch - - '{job="node-exporter"}' - - '{job="omero-server"}' - scheme: https - static_configs: - - targets: - - idr.openmicroscopy.org - labels: - prometheussrc: idr-production - - - job_name: federate-idr-analysis - basic_auth: - username: "{{ scrape_idr_username }}" - password: "{{ scrape_idr_password }}" - honor_labels: true - metrics_path: /prometheus/federate - params: - 'match[]': - # TODO: Decide what metrics to fetch - - '{__name__="node_cpu"}' - - '{__name__=~"node_memory.*"}' - - '{__name__="kube_pod_status_phase"}' - - '{__name__="container_cpu_user_seconds_total"}' - - '{__name__="container_memory_rss"}' - scheme: https - static_configs: - - targets: - - idr-analysis.openmicroscopy.org - labels: - prometheussrc: idr-analysis - - # The prometheus role sets the prometheus_internal_ip variable that is - # used later - - - role: idr.redmine_tracker - # This role sets the redmine_tracker_internal_ip variable - # Disable external port, access is via a proxy - redmine_tracker_port: 0 - - - role: ome.ssl_certificate - - - role: ome.nginx_proxy - # Just for configuration, well run Nginx in Docker - nginx_proxy_systemd_setup: false - - nginx_proxy_ssl: true - nginx_proxy_http2: true - nginx_proxy_ssl_certificate: "{{ ssl_certificate_bundled_path }}" - nginx_proxy_ssl_certificate_key: "{{ ssl_certificate_key_path }}" - nginx_proxy_force_ssl: True - nginx_proxy_conf_http: - - "client_max_body_size 128m" - nginx_proxy_sites: - # Default site - - nginx_proxy_is_default: true - nginx_proxy_backends: - - name: prometheus - location: /prometheus/ - server: http://prometheus:9090/ - additional: - - auth_basic Prometheus - - auth_basic_user_file /etc/nginx/prometheus.htpasswd - - name: grafana - location: /grafana/ - server: >- - http://grafana:3000/ - # idr-redmine (internal direct) - - nginx_proxy_server_name: idr-redmine-docker.openmicroscopy.org - nginx_proxy_backends: - - name: idr-redmine - location: / - server: http://redmine:3000/ - # idr-redmine (via external proxy) - - nginx_proxy_server_name: idr-redmine.openmicroscopy.org - nginx_proxy_backends: - - name: idr-redmine - location: / - server: http://redmine:3000/ - - tasks: - - - name: Run docker grafana - become: true - docker_container: - image: grafana/grafana:7.2.0 - env: - GF_SERVER_ROOT_URL: '%(protocol)s://%(domain)s:%(http_port)s/grafana/' - GF_SERVER_SERVE_FROM_SUB_PATH: 'true' - name: grafana - networks: - - name: monitoring - state: started - restart_policy: always - volumes: - - grafana-data:/var/lib/grafana - register: _grafana_container - - - name: prometheus htpasswd parent directory - become: true - file: - path: /etc/nginx - state: directory - mode: 0755 - - - name: prometheus htpasswd file - become: true - copy: - dest: /etc/nginx/prometheus.htpasswd - # default: admin:monitoring - content: >- - {{ - ome_monitoring_prometheus_htpasswd | - default('admin:$apr1$njrafrtU$19wf/I15zPuSudlM5Y50Z0') - }} - mode: 0644 - - # This is a dev Minio server so expose the port directly - - name: Run docker minio dev server - become: true - docker_container: - image: minio/minio:RELEASE.2020-10-18T21-54-12Z - command: gateway nas /data - env: - MINIO_ACCESS_KEY: "{{ ome_miniodev_access_key | default('minio') }}" - MINIO_SECRET_KEY: "{{ ome_miniodev_secret_key | default('minio123') }}" - name: miniodev - state: started - published_ports: - - '9000:9000' - restart_policy: always - volumes: - - "{{ nfs_minio_data_volume }}:/data" - - - name: Run docker nginx proxy - become: true - docker_container: - image: library/nginx:{{ nginx_version }} - name: nginx - networks: - - name: monitoring - - name: redmine - state: started - published_ports: - - '80:80' - - '443:443' - restart_policy: always - volumes: - - /etc/nginx/conf.d:/etc/nginx/conf.d:ro - - /etc/nginx/stream-conf.d:/etc/nginx/stream-conf.d:ro - - /etc/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - /etc/nginx/prometheus.htpasswd:/etc/nginx/prometheus.htpasswd:ro - - /etc/ssl/localcerts:/etc/ssl/localcerts:ro - - vars: - - monitored_node_exporter_hosts: "{{ ome_monitored_node_exporter_hosts | default([]) }}" - monitored_postgres_hosts: "{{ ome_monitored_postgres_hosts | default([]) }}" - monitored_omero_server_hosts: "{{ ome_monitored_omero_server_hosts | default([]) }}" - monitored_omero_web_hosts: "{{ ome_monitored_omero_web_hosts | default([]) }}" - - prometheus_docker_user: 909 - redmine_docker_user: 999 - - idr_internal_1: "{{ ome_monitored_idr_internal_1 | default('idr1') }}" - idr_internal_2: "{{ ome_monitored_idr_internal_2 | default('idr2') }}" - - scrape_username: "{{ ome_monitored_scrape_username | default('monitoring') }}" - scrape_password: "{{ ome_monitored_scrape_password | default('monitoring') }}" - scrape_idr_username: "{{ ome_monitored_scrape_idr_username | default('monitoring') }}" - scrape_idr_password: "{{ ome_monitored_scrape_idr_password | default('monitoring') }}" - - nginx_version: 1.18.0 - -# The following manual steps are required: -# 1. Login to Grafana with the default admin password -# 2. Set a new password (for ease set it to the same as the prometheus htpasswd) -# 3. Create a new prometheus datasource with url "http://prometheus:9090" -# (prometheus is the internal docker network container name) -# 4. Create your dashboards diff --git a/omedev/playbook.yml b/omedev/playbook.yml deleted file mode 100644 index 744030a3..00000000 --- a/omedev/playbook.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# OMEDEV servers -- name: Ome dev - hosts: omedev-docker - - roles: - - - role: ome.cli_utils - - - role: ome.versioncontrol_utils - - - role: ome.nfs_mount - # This will only have effect if nfs_share_mounts - # is defined in host/group vars - - - role: ome.docker diff --git a/omero/learning.yml b/omero/learning.yml deleted file mode 100644 index afa4dc74..00000000 --- a/omero/learning.yml +++ /dev/null @@ -1,201 +0,0 @@ -# Installation notes: -# - Set up OME RHEL 7 machine. -# - Run playbook. -# - systemctl stop omero-{server,web} -# - Migrate binary repository to /OMERO/ with rsync. -# - Migrate database with pg_dump, pg_restore. -# - Upgrade database. -# Do run shape_color_argb_to_rgba.sql. -# UPDATE shape SET points = -# REGEXP_REPLACE(REGEXP_REPLACE(points, '[^\[]*\[([^\]]*).*', -# '\1'), ', ', ' ', 'g') WHERE discriminator = 'polygon' AND points -# LIKE 'points[%'; -# - bin/omero db password for root and public users. -# - systemctl start omero-{server,web} - -- name: Learning Virtual Microscope playbook - hosts: omero-learning - roles: - - role: ome.lvm_partition - lvm_vgname: VolGroup00 - lvm_lvname: 'var_lib_psql' - lvm_lvmount: '/var/lib/pgsql' - lvm_lvsize: 20G - lvm_lvfilesystem: xfs - lvm_shrink: false - - - role: ome.postgresql - postgresql_databases: - - name: omero - postgresql_users: - - user: "{{ omero_server_dbuser | default('omero') }}" - password: "{{ omero_server_dbpassword | default('omero') }}" - databases: [omero] - - - role: ome.lvm_partition - lvm_vgname: VolGroup00 - lvm_lvname: 'omero' - lvm_lvmount: '/OMERO' - lvm_lvsize: 150G - lvm_lvfilesystem: xfs - lvm_shrink: false - - - role: ome.lvm_partition - lvm_vgname: VolGroup00 - lvm_lvname: 'opt_omero' - lvm_lvmount: '/opt/omero' - lvm_lvsize: 40G - lvm_lvfilesystem: xfs - lvm_shrink: false - - - role: ome.omero_server - omero_server_python_addons: - - "omero-py>={{ omero_py_release }}" - omero_server_config_set: - omero.client.ui.menu.dropdown.colleagues.enabled: false - omero.client.ui.menu.dropdown.everyone.label: "All courses" - omero.client.ui.menu.dropdown.leaders.label: "Courses:" - omero.db.poolsize: 100 - omero.jvmcfg.percent.blitz: 50 - omero.jvmcfg.percent.indexer: 20 - omero.jvmcfg.percent.pixeldata: 30 - omero.ldap.config: true - omero.ldap.base: "{{ omero_server_ldap_base }}" - omero.ldap.username: "{{ omero_server_ldap_username }}" - omero.ldap.password: "{{ omero_server_ldap_password | default('') }}" - omero.ldap.user_filter: "{{ omero_server_ldap_user_filter }}" - omero.ldap.new_user_group: "{{ omero_server_ldap_new_user_group }}" - omero.ldap.urls: "ldaps://{{ ldap_host }}:636" - omero.mail.config: true - omero.mail.from: "{{ omero_server_mail_from }}" - omero.mail.host: "{{ omero_server_mail_host }}" - omero.policy.binary_access: "+read,+write,-image,-plate" - omero.security.ignore_case: true - omero.security.keyStore: "/etc/pki/java/cacerts" - omero.security.trustStore: "/etc/pki/java/cacerts" - omero.security.keyStorePassword: "changeit" - omero.security.trustStorePassword: "changeit" - omero.sessions.timeout: 3600000 - omero_server_selfsigned_certificates: true - - - role: ome.redis - - - role: ome.nginx - - - role: ome.omero_web - omero_web_setup_nginx: false - omero_web_config_set: - omero.web.server_list: - - ["localhost", 4064, "Virtual Microscope"] - omero.web.prefix: '/dundee' - omero.web.static_url: '/dundee/static/' - omero.web.login_redirect: - redirect: - - webindex - viewname: "webindex_custom" - omero.web.ui.top_links: - - - "Virtual Microscope" - - "webindex" - - {"title": "Virtual Microscope"} - - - "HELP" - - "https://help.openmicroscopy.org/virtual-microscope.html" - - {"title": "Help", "target": "new"} - omero.web.ui.right_plugins: - - - "Acquisition" - - "webclient/data/includes/right_plugin.acquisition.js.html" - - "metadata_tab" - omero.web.caches: - default: - BACKEND: django_redis.cache.RedisCache - LOCATION: redis://127.0.0.1:6379/0 - omero.web.session_engine: django.contrib.sessions.backends.cache - omero.web.apps: - - "omero_gallery" - - "omero_iviewer" - - "virtualmicroscope" - omero.web.open_with: - - - "Image viewer" - - "webgateway" - - supported_objects: ["image"] - script_url: "webclient/javascript/ome.openwith_viewer.js" - - - "omero_iviewer" - - "omero_iviewer_index" - - supported_objects: ["images", "dataset", "well"] - script_url: "omero_iviewer/openwith.js" - label: "OMERO.iviewer" - omero.web.viewer.view: omero_iviewer.views.index - omero.web.public.enabled: true - omero.web.public.password: >- - {{ omero_web_public_password | default('public') }} - omero.web.public.url_filter: "/(webgateway|gallery)/" - omero.web.public.user: >- - {{ omero_web_public_user | default('public') }} - omero_web_apps_packages: - - omero-gallery=={{ omero_web_apps_release.omero_gallery }} - - omero-iviewer=={{ omero_web_apps_release.omero_iviewer }} - - "omero-virtual-microscope==\ - {{ omero_web_apps_release.omero_virtual_microscope }}" - omero_web_python_addons: - - "django-redis==5.0.0" - - "omero-py>={{ omero_py_release }}" - - - role: ome.postgresql_backup - postgresql_backup_compress: true - postgresql_backup_dir: /OMERO/pgbackup - postgresql_backup_filename_format: "nightly-omero-%a.pgdump.gz" - postgresql_backup_minimum_expected_size: 100000000 - - - tasks: - - name: Find OMERO.server log configuration - become: true - ansible.builtin.find: - paths: /opt/omero/server/OMERO.server/etc/ - patterns: "logback*.xml" - register: logbacks - - - name: OMERO.server logs are compressed on rollover - become: true - ansible.builtin.replace: - path: "{{ item.path }}" - regexp: "(\\\\$\\{om\ - ero\\.logfile\\}\\.\\%i)(\\<\\/fileNamePattern\\>)" - replace: "\\1.gz\\2" - backup: true - with_items: "{{ logbacks.files }}" - - - name: TLS certificate is installed for JVM - become: true - java_cert: - cert_url: "{{ ldap_host }}" - cert_port: 636 - keystore_path: "/etc/pki/java/cacerts" - keystore_pass: changeit - state: present - notify: restart omero-server - - - name: OMERO.web configuration is installed - become: true - copy: - src: "files/learning-omero-web.conf" - dest: "/etc/nginx/conf.d/omero-web.conf" - notify: restart nginx - - - name: OMERO.web starts on boot - become: true - ansible.builtin.service: - name: "{{ item }}.service" - enabled: true - loop: - - nginx - - omero-web - - vars: - postgresql_version: "13" - omero_server_release: 5.6.8 - omero_web_release: 5.22.1 - omero_py_release: "{{ omero_py_release_override | default('5.13.1') }}" - omero_web_apps_release: - omero_gallery: 3.4.2 - omero_iviewer: 0.12.0 - omero_virtual_microscope: 1.2.0 diff --git a/omero/nightshade-webclients.yml b/omero/nightshade-webclients.yml deleted file mode 100644 index 10542e21..00000000 --- a/omero/nightshade-webclients.yml +++ /dev/null @@ -1,185 +0,0 @@ -# Install OMERO.web with a public user on localhost - -- name: Ns webclients - hosts: ns-webclients - - roles: - - # Root LV Size - - role: ome.lvm_partition - tags: lvm - lvm_lvname: "{{ provision_root_lvname }}" - lvm_vgname: "{{ provision_root_vgname }}" - lvm_lvmount: / - lvm_lvsize: "{{ provision_rootsize }}" - lvm_lvfilesystem: "{{ provision_root_filesystem }}" - when: "not (molecule_test | default(False))" - - - role: ome.ssl_certificate - - - role: ome.nginx - - # OMERO.web configuration in host_vars in different repository - - role: ome.omero_web - omero_web_systemd_limit_nofile: 16384 - omero_web_python_addons: - - "omero-py>={{ omero_py_release }}" - - # Now OME are using RHEL without Spacewalk, the current best-method of - # checking `is server deployed in Dundee/SLS` is - # checking for the SLS nameservers. - - role: ome.system_monitor_agent - when: "'10.1.255.216' in ansible_dns.nameservers" - - handlers: - - name: Reload nginx - listen: ssl certificate changed - become: true - ansible.builtin.service: - name: nginx - state: reloaded - - - tasks: - - - name: Install open-vm-tools if system is a VMware vm - become: true - ansible.builtin.yum: - name: open-vm-tools - state: present - when: > - ((ansible_virtualization_type is defined) - and (ansible_virtualization_type == "VMware")) - and not (molecule_test | default(False)) - - # (Total cores / 2), leaving some for WSGI - # post 2.3 'dest' should be renamed 'path' - - name: NGINX - Performance tuning - worker processes - become: true - ansible.builtin.replace: - dest: "/etc/nginx/nginx.conf" - regexp: '^worker_processes\s+\d+;' - replace: >- - worker_processes {{ ((ansible_processor_count * - ansible_processor_cores) / 2) | round | int }}; - - # post 2.3 'dest' should be renamed 'path' - # cf https://www.digitalocean.com/community/tutorials/ - # how-to-optimize-nginx-configuration - - name: NGINX - Performance tuning - worker connections - become: true - ansible.builtin.replace: - dest: "/etc/nginx/nginx.conf" - regexp: 'worker_connections\s+\d+;' - replace: "worker_connections 65000;" - - - name: NGINX - create nested includes directory - become: true - ansible.builtin.file: - path: /etc/nginx/conf.d-nested-includes - state: directory - mode: 0755 - - - name: NGINX - SSL Configuration - become: true - template: - src: templates/nginx-confdnestedincludes-ssl-conf.j2 - dest: /etc/nginx/conf.d-nested-includes/ssl.conf - mode: 0644 - notify: - - restart nginx - - - name: NGINX - Custom Paper Redirect - become: true - template: - src: templates/nginx-confdnestedincludes-ns-pub-redirects-conf.j2 - dest: /etc/nginx/conf.d-nested-includes/ns-pub-redirects.conf - mode: 0644 - notify: - - restart nginx - - vars: - omero_web_config_set_for_playbook: - omero.web.nginx_server_extra_config: - - 'include /etc/nginx/conf.d-nested-includes/*.conf;' - omero_web_config_set: >- - {{ - omero_web_config_set_for_playbook | combine( - (omero_web_config_set_for_group | default({})), - (omero_web_config_set_for_host | default({}))) - }} - - omero_web_release: "{{ omero_web_release_override | default('5.22.1') }}" - omero_py_release: "{{ omero_py_release_override | default('5.15.0') }}" - omero_figure_release: >- - {{ omero_figure_release_override | default('6.0.1') }} - omero_fpbioimage_release: >- - {{ omero_fpbioimage_release_override | default('0.4.1') }} - omero_iviewer_release: >- - {{ omero_iviewer_release_override | default('0.13.0') }} - omero_parade_release: >- - {{ omero_parade_release_override | default('0.2.4') }} - omero_webtagging_autotag_release: >- - {{ omero_webtagging_autotag_release_override | default('3.2.0') }} - omero_webtagging_tagsearch_release: >- - {{ omero_webtagging_tagsearch_release_override | default('3.2.0') }} - - omero_web_apps_names: - - omero_figure - - omero_fpbioimage - - omero_iviewer - - omero_parade - - omero_webtagging_autotag - - omero_webtagging_tagsearch - - omero_web_apps_packages: - - "omero-figure=={{ omero_figure_release }}" - - "omero-fpbioimage=={{ omero_fpbioimage_release }}" - - "omero-iviewer=={{ omero_iviewer_release }}" - - "omero-parade=={{ omero_parade_release }}" - - "omero-webtagging-autotag=={{ omero_webtagging_autotag_release }}" - - "omero-webtagging-tagsearch=={{ omero_webtagging_tagsearch_release }}" - - omero_web_apps_top_links: - - label: Figure - link: figure_index - attrs: - title: Open Figure in new tab - target: _blank - - label: Tag Search - link: tagsearch - - omero_web_apps_config_append: - omero.web.open_with: - - - omero_figure - - new_figure - - supported_objects: - - images - target: _blank - label: OMERO.figure - - - omero_fpbioimage - - fpbioimage_index - - supported_objects: - - image - script_url: fpbioimage/openwith.js - label: FPBioimage - - - omero_iviewer - - omero_iviewer_index - - supported_objects: - - images - - dataset - - well - script_url: omero_iviewer/openwith.js - label: OMERO.iviewer - omero.web.ui.center_plugins: - - - Auto Tag - - omero_webtagging_autotag/auto_tag_init.js.html - - auto_tag_panel - - - Parade - - omero_parade/init.js.html - - omero_parade - - omero_web_apps_config_set: - omero.web.viewer.view: omero_iviewer.views.index - - nginx_version: 1.18.0 diff --git a/omero/ome-demoserver.yml b/omero/ome-demoserver.yml index 99fc6085..a59bcc51 100644 --- a/omero/ome-demoserver.yml +++ b/omero/ome-demoserver.yml @@ -8,106 +8,24 @@ pre_tasks: - name: Install open-vm-tools if system is a VMware vm become: true - ansible.builtin.yum: + ansible.builtin.dnf: name: open-vm-tools state: installed when: > ((ansible_virtualization_type is defined) and (ansible_virtualization_type == "VMware")) - # Perhaps alter the role at - # https://github.com/openmicroscopy/ansible-role-lvm-partition/ - # to make some of the variables non-required. - - name: Resize root FS without altering mount options - tags: lvm - become: true - lvol: - lv: root - vg: VolGroup00 - size: "{{ provision_root_lvsize }}" - shrink: false - - - name: Install Make Movie script Prerequisite | MEncoder - Repo - become: true - ansible.builtin.yum: - name: "http://li.nux.ro/download/nux/dextop/el7\ - /x86_64/nux-dextop-release-0-5.el7.nux.noarch.rpm" - state: present - - - name: Install Make Movie script Prerequisite | MEncoder - Package - become: true - ansible.builtin.yum: - name: mencoder - state: present - - - name: Server-side script prerequisites - become: true - ansible.builtin.yum: - name: "{{ item }}" - state: present - with_items: - - mencoder # For the 'make movie' script - roles: - # Now OME are using RHEL without Spacewalk, the current best-method of - # checking `is server deployed in Dundee/SLS` is - # checking for the SLS nameservers. - - role: ome.system_monitor_agent - tags: monitoring - when: "'10.1.255.216' in ansible_dns.nameservers" - - # Disk Layout - PostgreSQL | data dir on separate VG (SSD) - - role: ome.lvm_partition - tags: lvm - lvm_lvname: pgdata - lvm_vgname: "{{ provision_postgres_vgname }}" - lvm_lvmount: /var/lib/pgsql - lvm_lvsize: "{{ provision_postgres_lvsize }}" - lvm_lvfilesystem: "{{ filesystem }}" - lvm_shrink: false - - # Disk Layout - OMERO | VG and LV (separate disk) for Binary Repository - - role: ome.lvm_partition - tags: lvm - lvm_lvname: datadir - lvm_vgname: "{{ provision_omero_server_datadir_vgname }}" - lvm_lvmount: "{{ omero_server_datadir }}" - lvm_lvsize: "{{ provision_omero_server_datadir_lvsize }}" - lvm_lvfilesystem: "{{ filesystem }}" - lvm_shrink: false - - # Disk Layout - OMERO.server | LV for dist & logs - - role: ome.lvm_partition - tags: lvm - lvm_lvname: omero_server_basedir - lvm_vgname: VolGroup00 - lvm_lvmount: "{{ omero_server_basedir }}" - lvm_lvsize: "{{ provision_omero_server_basedir_lvsize }}" - lvm_lvfilesystem: "{{ filesystem }}" - lvm_shrink: false - - # Disk Layout - OMERO.web | LV for dist & logs - - role: ome.lvm_partition - tags: lvm - lvm_lvname: omero_web_basedir - lvm_vgname: VolGroup00 - lvm_lvmount: "{{ omero_web_basedir }}" - lvm_lvsize: "{{ provision_omero_web_basedir_lvsize }}" - lvm_lvfilesystem: "{{ filesystem }}" - lvm_shrink: false - - - role: ome.nginx - nginx_version: 1.16.1 - role: ome.postgresql # no_log: true postgresql_databases: - - name: omero + - name: omero + owner: demo postgresql_users: - - user: "{{ omero_server_dbuser | default('omero') }}" - password: "{{ omero_server_dbpassword | default('omero') }}" - databases: - - omero + - user: "{{ omero_server_dbuser | default('omero') }}" + password: "{{ omero_server_dbpassword | default('omero') }}" + databases: [] - role: ome.omero_server # Defaults overridden in private configuration @@ -117,15 +35,17 @@ omero_server_dbname: omero omero_server_systemd_limit_nofile: 16384 + - role: ome.nginx + - role: ome.omero_web # Defaults overridden in private configuration omero_web_systemd_limit_nofile: 16384 omero_web_python_addons: - "omero-figure=={{ omero_figure_release }}" - "omero-fpbioimage=={{ omero_fpbioimage_release }}" - - "omero-webtagging-autotag=={{ omero_webtagging_autotag_release }}" - - "omero-webtagging-tagsearch==\ - {{ omero_webtagging_tagsearch_release }}" + - "omero-autotag=={{ omero_autotag_release }}" + - "omero-tagsearch==\ + {{ omero_tagsearch_release }}" - "omero-iviewer=={{ omero_iviewer_release }}" - "omero-parade=={{ omero_parade_release }}" - "omero-signup=={{ omero_signup_release }}" @@ -156,7 +76,6 @@ postgresql_backup_compress: true postgresql_backup_dir: /OMERO/pgbackup postgresql_backup_filename_format: "nightly-omero-%a.pgdump.gz" - postgresql_backup_minimum_expected_size: 100000000 handlers: - name: Reload web server @@ -168,6 +87,10 @@ post_tasks: + - name: Allow nginx to connect to omero-web + become: yes + command: setsebool -P httpd_can_network_connect on + - name: NGINX - Performance tuning - worker processes become: true ansible.builtin.replace: @@ -256,51 +179,6 @@ - restart omero-web no_log: true - - name: Check_MK postgres plugin | check for plugin existence - tags: monitoring - ansible.builtin.stat: - path: "{{ check_mk_agent_plugin_path }}/mk_postgres" - register: check_mk_postgres_plugin_st - - - name: Check_MK postgres plugin | activate the plugin - tags: monitoring - become: true - command: > - cp "{{ check_mk_agent_plugin_path }}/mk_postgres" - /usr/share/check-mk-agent/plugins/ - creates=/usr/share/check-mk-agent/plugins/mk_postgres - when: check_mk_postgres_plugin_st.stat.exists - - - name: Check_MK logwatch plugin | check for plugin existence - tags: monitoring - ansible.builtin.stat: - path: "{{ check_mk_agent_plugin_path }}/mk_logwatch" - register: check_mk_logwatch_plugin_st - - - name: Check_MK logwatch plugin | activate the plugin - tags: monitoring - become: true - command: > - cp "{{ check_mk_agent_plugin_path }}/mk_logwatch" - /usr/share/check-mk-agent/plugins/ - creates=/usr/share/check-mk-agent/plugins/mk_logwatch - when: check_mk_logwatch_plugin_st.stat.exists - - - name: Check_MK logwatch plugin | check for default config file - tags: monitoring - ansible.builtin.stat: - path: "{{ check_mk_agent_config_example_path }}/logwatch.cfg" - register: check_mk_logwatch_plugin_conf_st - - - name: Check_MK logwatch plugin | copy the default config - tags: monitoring - become: true - command: > - cp "{{ check_mk_agent_config_example_path }}/logwatch.cfg" - "{{ check_mk_agent_config_path }}/logwatch.cfg" - creates="{{ check_mk_agent_config_path }}/logwatch.cfg" - when: check_mk_logwatch_plugin_conf_st.stat.exists - - name: PostgreSQL Nightly Backups | Remove old cron job become: true ansible.builtin.file: @@ -332,26 +210,26 @@ vars: omero_figure_release: >- - {{ omero_figure_release_override | default('6.0.1') }} + {{ omero_figure_release_override | default('6.2.2') }} omero_figure_script_release: >- - {{ omero_figure_script_release_override | default('v6.0.1') }} + {{ omero_figure_script_release_override | default('v6.2.2') }} omero_fpbioimage_release: >- {{ omero_fpbioimage_release_override | default('0.4.1') }} omero_iviewer_release: >- - {{ omero_iviewer_release_override | default('0.13.0') }} + {{ omero_iviewer_release_override | default('0.14.0') }} omero_parade_release: >- {{ omero_parade_release_override | default('0.2.4') }} - omero_webtagging_autotag_release: >- - {{ omero_webtagging_autotag_release_override | default('3.2.0') }} - omero_webtagging_tagsearch_release: >- - {{ omero_webtagging_tagsearch_release_override | default('3.2.0') }} + omero_autotag_release: >- + {{ omero_autotag_release_override | default('4.0.1') }} + omero_tagsearch_release: >- + {{ omero_tagsearch_release_override | default('4.1.1') }} omero_signup_release: >- - {{ omero_signup_release_override | default('0.3.2') }} + {{ omero_signup_release_override | default('0.3.3') }} omero_server_release: >- - {{ omero_server_release_override | default('5.6.8') }} - omero_web_release: "{{ omero_web_release_override | default('5.22.1') }}" - omero_py_release: "{{ omero_py_release_override | default('5.15.0') }}" + {{ omero_server_release_override | default('5.6.11') }} + omero_web_release: "{{ omero_web_release_override | default('5.26.0') }}" + omero_py_release: "{{ omero_py_release_override | default('5.19.2') }}" # For https://github.com/openmicroscopy/ansible-role-java, # which is a dependency. java_jdk_install: true @@ -388,7 +266,7 @@ [1] In your browser, go to omero-guides.readthedocs.io/en/latest and click on OMERO walkthrough example under Getting started.' - postgresql_version: "11" + postgresql_version: "16" filesystem: "xfs" omero_server_config_set: @@ -445,7 +323,7 @@ omero.web.wsgi_workers: >- {{ (2 * (ansible_processor_count * ansible_processor_cores)) + 1 }} - omero.web.admins: "{{ omero_web_admins }}" + # omero.web.admins: "{{ omero_web_admins }}" # https://pypi.org/project/omero-iviewer/ - set iviewer to default viewer omero.web.viewer.view: omero_iviewer.views.index omero.web.nginx_server_extra_config: diff --git a/omero/ome-dundeeomero.yml b/omero/ome-dundeeomero.yml deleted file mode 100644 index 7ec07097..00000000 --- a/omero/ome-dundeeomero.yml +++ /dev/null @@ -1,278 +0,0 @@ -# Install OMERO.server and prepare the OME (UoD/SLS) prerequisites - -- name: Dundeeomero server aka nightshade - hosts: ome-dundeeomero.openmicroscopy.org - pre_tasks: - - name: Install open-vm-tools if system is a VMware vm - become: true - ansible.builtin.yum: - name: open-vm-tools - state: present - when: > - ((ansible_virtualization_type is defined) - and (ansible_virtualization_type == "VMware")) - and not (molecule_test | default(False)) - - # Perhaps alter the role at - # https://github.com/openmicroscopy/ansible-role-lvm-partition/ - # to make some of the variables non-required. - - name: Resize root FS without altering mount options - tags: lvm - become: true - lvol: - lv: root - vg: rhel - size: "{{ provision_root_lvsize }}" - when: "not (molecule_test | default(False))" - - - name: Install Make Movie script Prerequisite | MEncoder - Repo - become: true - ansible.builtin.yum: - name: "http://li.nux.ro/download/nux/dextop/el7\ - /x86_64/nux-dextop-release-0-5.el7.nux.noarch.rpm" - state: latest - - # web server is included for decoupled OMERO.web - - name: OMERO.figure server-side&script prerequisites & web server - become: true - ansible.builtin.yum: - name: "{{ item }}" - state: present - with_items: - # For OMERO.figure - - mencoder # For the 'make movie' script - - roles: - # Disk Layout - PostgreSQL | data dir - - role: ome.lvm_partition - tags: lvm - lvm_lvname: pgdata - lvm_vgname: postgresql - lvm_lvmount: /var/lib/pgsql - lvm_lvsize: "{{ provision_postgres_lvsize }}" - lvm_lvfilesystem: "{{ filesystem }}" - when: "not (molecule_test | default(False))" - - # Disk Layout - OMERO | data dir - - role: ome.lvm_partition - tags: lvm - lvm_lvname: basedir - lvm_vgname: omero - lvm_lvmount: "{{ omero_common_basedir }}" - lvm_lvsize: "{{ provision_omero_basedir_lvsize }}" - lvm_lvfilesystem: "{{ filesystem }}" - when: "not (molecule_test | default(False))" - - # Mock database user & creds, to allow Playbook to install - # OMERO, and allow for a manual PostgresSQL dump/restore. - - role: ome.postgresql - postgresql_databases: - - name: "{{ omero_server_dbname | default('omero') }}" - postgresql_users: - - user: "{{ omero_server_dbuser | default('omero') }}" - password: "{{ omero_server_dbpassword | default('omero') }}" - databases: - - "{{ omero_server_dbname | default('omero') }}" - - - # Note - had to have these set to `install-mock` to progress role - # installation before changing config to restored DB from other system. - - role: ome.omero_server - omero_server_release: 5.6.8 - omero_server_datadir_manage: "{{ molecule_test | default(False) }}" - omero_server_systemd_limit_nofile: 16384 - omero_server_systemd_after: >- - {{ molecule_test | default(False) | ternary([], ['gpfs.service']) }} - omero_server_systemd_requires: >- - {{ molecule_test | default(False) | ternary([], ['gpfs.service']) }} - omero_server_system_user_manage: "{{ molecule_test | default(False) }}" - - # Current server proxies to a decoupled OMERO.web server - # Initially replicate this setup, to minimise changes. - - role: ome.nginx - - - role: ome.ssl_certificate - - - role: ome.postgresql_backup - postgresql_backup_compress: true - postgresql_backup_dir: >- - {{ omero_server_db_dumpdir_parent | default('/tmp/pgbackup') }} - postgresql_backup_filename_format: "nightly-omero-%a.pgdump.gz" - postgresql_backup_minimum_expected_size: 100000000 - - - handlers: - - name: Reload nginx - listen: ssl certificate changed - become: true - ansible.builtin.service: - name: nginx - state: reloaded - - tasks: - - # OMERO doesn't limit sizes and fills up /tmp - # thus we need to create another tmp - - name: Create another temporary directory - become: true - ansible.builtin.file: - path: "{{ omero_server_systemd_environment.OMERO_TMPDIR }}" - state: directory - mode: 0700 - owner: "{{ omero_server_system_user }}" - - - name: NGINX - enable service / start on boot - become: true - ansible.builtin.systemd: - name: nginx - enabled: true - - # post 2.3 'dest' should be renamed 'path' - - name: NGINX - Performance tuning - worker processes - become: true - ansible.builtin.replace: - dest: "/etc/nginx/nginx.conf" - regexp: '^worker_processes\s+\d+;' - replace: "worker_processes 1;" - notify: - - restart nginx - - # post 2.3 'dest' should be renamed 'path' - # cf https://www.digitalocean.com/community/tutorials/how - # -to-optimize-nginx-configuration - - name: NGINX - Performance tuning - worker connections - become: true - ansible.builtin.replace: - dest: "/etc/nginx/nginx.conf" - regexp: 'worker_connections\s+\d+;' - replace: "worker_connections 65000;" - notify: - - restart nginx - - - name: NGINX - create nested includes directory - become: true - ansible.builtin.file: - path: /etc/nginx/conf.d-nested-includes - state: directory - mode: 0755 - - # post 2.3 'destfile' should be renamed 'path' - - name: NGINX - Configuration - become: true - template: - src: nginx-omero.conf.j2 - dest: /etc/nginx/conf.d/omero-web.conf - mode: 0644 - notify: - - restart nginx - - - name: PostgreSQL Nightly Backups | Remove old cron job - become: true - ansible.builtin.file: - path: /etc/cron.daily/nightly-pg_dump-omero.sh - state: absent - - - name: Create a figure scripts directory - become: true - ansible.builtin.file: - path: /opt/omero/server/OMERO.server/lib/scripts/omero/figure_scripts - state: directory - mode: 0755 - recurse: true - owner: root - - - name: Download the Figure_To_Pdf.py script - become: true - ansible.builtin.get_url: - url: "https://raw.githubusercontent.com/ome\ - /omero-figure\ - /v{{ omero_figure_release }}/omero_figure/scripts\ - /omero/figure_scripts/Figure_To_Pdf.py" - dest: "/opt/omero/server/OMERO.server/lib/scripts\ - /omero/figure_scripts/Figure_To_Pdf.py" - mode: 0644 - owner: root - force: true - - - name: Download the Dataset_Images_To_New_Figure.py script - become: true - ansible.builtin.get_url: - url: "https://raw.githubusercontent.com/ome\ - /omero-guide-figure\ - /f45f733a16852ae8b3c52ec93aef480d26b8e9f9/scripts/Dataset\ - _Images_To_New_Figure.py" - dest: "/opt/omero/server/OMERO.server/lib/scripts\ - /omero/figure_scripts/Dataset_Images_To_New_Figure.py" - mode: 0644 - owner: root - force: true - - - name: Download the Figure_Images_To_Dataset.py script - become: true - ansible.builtin.get_url: - url: "https://raw.githubusercontent.com/ome\ - /omero-guide-figure\ - /f45f733a16852ae8b3c52ec93aef480d26b8e9f9/scripts/Figure\ - _Images_To_Dataset.py" - dest: "/opt/omero/server/OMERO.server/lib/scripts\ - /omero/figure_scripts/Figure_Images_To_Dataset.py" - mode: 0644 - owner: root - force: true - - vars: - # For https://github.com/openmicroscopy/ansible-role-java - # which is a dependency. - java_jdk_install: true - - nginx_version: 1.18.0 - postgresql_version: "11" - filesystem: "xfs" - omero_figure_release: >- - {{ omero_figure_release_override | default('6.0.1') }} - omero_py_release: "{{ omero_py_release_override | default('5.15.0') }}" - - omero_server_config_set_production: - omero.db.poolsize: 60 - omero.fs.repo.path: >- - %user%_%userId%/%thread%//%year%-%month%/%day%/%time% - omero.jvmcfg.percent.blitz: 50 - omero.jvmcfg.percent.indexer: 20 - omero.jvmcfg.percent.pixeldata: 20 - omero.jvmcfg.system_memory: 17000 - omero.ldap.base: "{{ omero_server_ldap_base | default('example') }}" - omero.ldap.config: true - omero.ldap.urls: >- - {{ omero_server_ldap_urls | default('ldap://example.org') }} - omero.mail.config: true - omero.mail.from: >- - {{ omero_server_mail_from | default('omero@example.org') }} - omero.mail.host: >- - {{ omero_server_mail_host | default('smtp.example.org') }} - omero.ldap.new_user_group: "My Data" - omero.search.batch: 100 - omero.security.password_provider: chainedPasswordProvider431 - omero.throttling.method_time.error: 60000 - omero.Ice.Default.Host: >- - {{ omero_server_ice_default_host | default('127.0.0.1') }} - Ice.Admin.Endpoints: >- - {{ omero_server_ice_admin_endpoints | default('tcp -h 127.0.0.1') }} - omero.data.dir: "{{ omero_server_datadir | default('/OMERO') }}" - - omero_server_selfsigned_certificates: true - - # Production config can't be tested in molecule - omero_server_config_set: >- - {{ molecule_test | default(False) | - ternary({}, omero_server_config_set_production) }} - omero_server_python_addons: - # For OMERO.figure script - - "reportlab<3.6" - - markdown - - "omero-py>={{ omero_py_release }}" - - # Workaround lack of restriction on temp file sizes - # https://github.com/ome/omero-web/issues/118 - # The downside is that it won't be automatically cleared out - omero_server_systemd_environment: - OMERO_TMPDIR: /opt/omero/server/tmp diff --git a/omero/omero-firewall.yml b/omero/omero-firewall.yml deleted file mode 100644 index 2d829768..00000000 --- a/omero/omero-firewall.yml +++ /dev/null @@ -1,68 +0,0 @@ -# Setup up iptables firewall on OMERO servers - -- name: Firewall - hosts: monitored - - roles: - - - role: ome.iptables_raw - - tasks: - - # Allow: - # - all established/related in/out - # - all internal localhost connections - # - ICMP echo (ping) - # - ssh incoming connections - - name: Iptables ssh and related - become: true - iptables_raw_25: - name: ssh_and_established - keep_unmanaged: false - rules: | - -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT - -A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT - -A INPUT -i lo -j ACCEPT - -A INPUT -p icmp --icmp-type echo-request -j ACCEPT - -A INPUT -p tcp -m tcp --dport 22 -j ACCEPT - state: present - # Highest priority - weight: 0 - - # Use a low priority REJECT rule so that clients can detect when - # they've been rejected - # The alternative of setting a default DROP policy will leave them - # hanging until they timeout, though this may be preferable for public - # servers: - # http://www.chiark.greenend.org.uk/~peterb/network/drop-vs-reject - - name: Iptables default - become: true - iptables_raw_25: - name: default_rules - rules: | - -A INPUT -j REJECT - -A FORWARD -j REJECT - -A OUTPUT -j ACCEPT - state: present - # Lowest priority - weight: 99 - - # All other ports that allow incoming connections: - # - web - # - omero - # - GPFS - # - Check_MK - - name: Iptables OME ports - become: true - iptables_raw_25: - name: ome_ports - rules: | - -A INPUT -p tcp -m multiport --dports 80,443 -j ACCEPT - -A INPUT -p tcp -m multiport --dports 4063,4064 -j ACCEPT - {% for s in gpfs_cluster_source | default([]) %} - -A INPUT -p tcp -m tcp --dport 1191 -s {{ s }} -j ACCEPT - {% endfor %} - {% if (checkmk_server_source | default('')) %} - -A INPUT -p tcp -m tcp --dport 6556 -s {{ checkmk_server_source }} -j ACCEPT - {% endif %} - state: present diff --git a/omero/omero-monitoring-agents.yml b/omero/omero-monitoring-agents.yml deleted file mode 100644 index 5a513ef5..00000000 --- a/omero/omero-monitoring-agents.yml +++ /dev/null @@ -1,115 +0,0 @@ -# Setup prometheus agents - -- name: Monitoring agents - hosts: omero-server - - roles: - - - role: ome.prometheus_jmx - - - role: ome.prometheus_postgres - prometheus_postgres_dbname: omero - - # For restart handlers - - role: ome.omero_common - - - role: ome.omero_prometheus_exporter - omero_prometheus_exporter_omero_user: >- - {{ secret_omero_prometheus_exporter_omero_user | default('root') }} - omero_prometheus_exporter_omero_password: >- - {{ secret_omero_prometheus_exporter_omero_password - | default('omero') }} - - tasks: - - - name: Omero-server prometheus jmx agents - become: true - copy: - dest: "{{ omero_common_basedir }}/server/config/prometheus.omero" - src: omero-server-config-prometheus.omero - mode: 0644 - notify: - - restart omero-server - -- name: OMERO web - hosts: omero-web - - roles: - - - role: ome.omero_web_django_prometheus - - -# NOTE: This assumes omero-web.conf is present and includes -# /etc/nginx/conf.d-nested-includes -- name: Monitoring - hosts: monitored - - roles: - - - role: ome.prometheus_node - - # Autodetect whether selinux is enabled - - role: ome.selinux_utils - - tasks: - - - name: Nginx selinux allow network connect - become: true - seboolean: - name: httpd_can_network_connect - state: true - persistent: true - when: selinux_enabled - - - name: Create monitoring htpasswd - become: true - copy: - content: >- - {{ secret_monitoring_nginx_htpasswd | - default(monitoring_nginx_htpasswd) }} - dest: /etc/nginx/monitoring.htpasswd - mode: 0644 - - # This is fine to apply to all servers because if an exporter doesn't - # exist it will return an error, and we know which exporters to expect - # when scraping - - name: Create nginx proxy for prometheus exporters - become: true - copy: - dest: /etc/nginx/conf.d-nested-includes/proxy-exporters.conf - src: confd-nested-proxy-exporters.conf - mode: 0644 - notify: - - restart nginx - - handlers: - - name: restart nginx - become: true - service: - name: nginx - state: restarted - - vars: - # monitoring:monitoring - monitoring_nginx_htpasswd: | - monitoring:$apr1$njrafrtU$19wf/I15zPuSudlM5Y50Z0 - -- name: OMERO-web - hosts: omero-web - - tasks: - - name: Create nginx proxy for prometheus web exporters - become: true - copy: - dest: /etc/nginx/conf.d-nested-includes/proxy-exporters-web.conf - src: confd-nested-proxy-exporters-web.conf - mode: 0644 - notify: - - restart nginx - - handlers: - - name: restart nginx - become: true - service: - name: nginx - state: restarted diff --git a/omero/roles/.gitignore b/omero/roles/.gitignore new file mode 100644 index 00000000..46724a47 --- /dev/null +++ b/omero/roles/.gitignore @@ -0,0 +1,2 @@ +roles +.vagrant diff --git a/omero/sls-gallery.yml b/omero/sls-gallery.yml deleted file mode 100644 index 6c817b46..00000000 --- a/omero/sls-gallery.yml +++ /dev/null @@ -1,171 +0,0 @@ -# Installation notes: -# - Set up OME RHEL 7 machine. -# - Run playbook. -# - systemctl stop omero-{server,web} -# - Migrate binary repository to /OMERO/ with rsync. -# - Migrate database with pg_dump, pg_restore. -# - Upgrade database. -# - bin/omero db password for root user. -# - systemctl start omero-{server,web} - -- name: Sls gallery - hosts: omero-sls-gallery - roles: - - role: ome.lvm_partition - lvm_vgname: VolGroup00 - lvm_lvname: 'var_lib_psql' - lvm_lvmount: '/var/lib/pgsql' - lvm_lvsize: 8G - lvm_lvfilesystem: xfs - lvm_shrink: false - - - role: ome.postgresql - postgresql_databases: - - name: omero - postgresql_users: - - user: "{{ omero_server_dbuser | default('omero') }}" - password: "{{ omero_server_dbpassword | default('omero') }}" - databases: [omero] - - - role: ome.postgresql_backup - postgresql_backup_compress: true - postgresql_backup_dir: /OMERO/pgbackup - postgresql_backup_filename_format: "nightly-omero-%a.pgdump.gz" - postgresql_backup_minimum_expected_size: 100000000 - - - role: ome.lvm_partition - lvm_vgname: VolGroup00 - lvm_lvname: 'omero' - lvm_lvmount: '/OMERO' - lvm_lvsize: 80G - lvm_lvfilesystem: xfs - lvm_shrink: false - - - role: ome.lvm_partition - lvm_vgname: VolGroup00 - lvm_lvname: 'opt_omero' - lvm_lvmount: '/opt/omero' - lvm_lvsize: 40G - lvm_lvfilesystem: xfs - lvm_shrink: false - - - role: ome.omero_server - omero_server_config_set: - omero.client.ui.menu.dropdown.colleagues.enabled: false - omero.client.ui.menu.dropdown.everyone.label: "All images" - omero.client.ui.menu.dropdown.leaders.label: "Gallery:" - omero.db.poolsize: 50 - omero.jvmcfg.percent.blitz: 50 - omero.jvmcfg.percent.indexer: 20 - omero.jvmcfg.percent.pixeldata: 30 - omero.ldap.config: true - omero.ldap.base: "{{ omero_server_ldap_base }}" - omero.ldap.username: "{{ omero_server_ldap_username }}" - omero.ldap.user_filter: "{{ omero_server_ldap_user_filter }}" - omero.ldap.group_filter: "{{ omero_server_ldap_group_filter }}" - omero.ldap.group_mapping: "{{ omero_server_ldap_group_mapping }}" - omero.ldap.new_user_group: "{{ omero_server_ldap_new_user_group }}" - omero.ldap.urls: "ldap://{{ ldap_host }}:389" - omero.mail.config: true - omero.mail.from: "{{ omero_server_mail_from }}" - omero.mail.host: "{{ omero_server_mail_host }}" - omero.pixeldata.max_plane_height: 5120 - omero.pixeldata.max_plane_width: 5120 - omero_server_selfsigned_certificates: true - - - role: ome.redis - - - role: ome.nginx - - - role: ome.omero_web - omero_web_setup_nginx: false - omero_web_systemd_start: true - omero_web_config_set: - omero.web.server_list: - - ["localhost", 4064, "SLS Gallery"] - omero.web.prefix: '/ome-sls' - omero.web.static_url: '/ome-sls/static/' - omero.web.login_redirect: - redirect: - - webindex - viewname: "load_template" - query_string: "experimenter=-1" - args: - - userdata - omero.web.ui.top_links: - - - "Image Gallery" - - "webindex" - - title: "Image Gallery" - - - "HELP" - - "https://help.openmicroscopy.org/web-client.html" - - title: "Help" - target": "new" - - - "SLS Homepage" - - "https://www.lifesci.dundee.ac.uk/" - - title: "SLS Homepage" - target: "new" - omero.web.caches: - default: - BACKEND: django_redis.cache.RedisCache - LOCATION: redis://127.0.0.1:6379/0 - omero.web.session_engine: django.contrib.sessions.backends.cache - omero.web.apps: - - "omero_iviewer" - omero.web.open_with: - - - "Image viewer" - - "webgateway" - - supported_objects: ["image"] - script_url: "webclient/javascript/ome.openwith_viewer.js" - - - "omero_iviewer" - - "omero_iviewer_index" - - supported_objects": ["images", "dataset", "well"] - script_url": "omero_iviewer/openwith.js" - label: "OMERO.iviewer" - omero.web.viewer.view: omero_iviewer.views.index - omero_web_apps_packages: - - omero-iviewer=={{ omero_web_apps_release.omero_iviewer }} - omero_web_python_addons: - - "django-redis==5.0.0" - - "omero-py>={{ omero_py_release }}" - - tasks: - - name: Find OMERO.server log configuration - become: true - ansible.builtin.find: - paths: /opt/omero/server/OMERO.server/etc/ - patterns: "logback*.xml" - register: logbacks - - - name: OMERO.server logs are compressed on rollover - become: true - replace: - path: "{{ item.path }}" - regexp: "(\\\\$\\{om\ - ero\\.logfile\\}\\.\\%i)(\\<\\/fileNamePattern\\>)" - replace: "\\1.gz\\2" - backup: true - with_items: "{{ logbacks.files }}" - - - name: OMERO.web configuration is installed - become: true - copy: - src: "files/sls-gallery-omero-web.conf" - dest: "/etc/nginx/conf.d/omero-web.conf" - notify: restart nginx - - - name: OMERO.web starts on boot - become: true - ansible.builtin.service: - name: "{{ item }}.service" - enabled: true - loop: - - nginx - - omero-web - - vars: - postgresql_version: "13" - omero_server_release: 5.6.3 - omero_web_release: 5.15.0 - omero_web_apps_release: - omero_iviewer: 0.11.3 - omero_py_release: "{{ omero_py_release_override | default('5.12.0') }}" diff --git a/omero/templates/omero-web-config-for-webapps.j2 b/omero/templates/omero-web-config-for-webapps.j2 index 9f770dfb..58f42424 100644 --- a/omero/templates/omero-web-config-for-webapps.j2 +++ b/omero/templates/omero-web-config-for-webapps.j2 @@ -19,11 +19,11 @@ config append -- omero.web.open_with '["omero_iviewer", "omero_iviewer_index", { config set -- omero.web.viewer.view {{ omeroweb_default_viewer_override | default('omero_iviewer.views.index') }} # Autotag -config append -- omero.web.apps '"omero_webtagging_autotag"' -config append -- omero.web.ui.center_plugins '["Auto Tag", "omero_webtagging_autotag/auto_tag_init.js.html", "auto_tag_panel"]' +config append -- omero.web.apps '"omero_autotag"' +config append -- omero.web.ui.center_plugins '["Auto Tag", "omero_autotag/auto_tag_init.js.html", "auto_tag_panel"]' # tagsearch -config append -- omero.web.apps '"omero_webtagging_tagsearch"' +config append -- omero.web.apps '"omero_tagsearch"' config append -- omero.web.ui.top_links '["Tag Search", "tagsearch"]' # parade diff --git a/omero/training-server/idr_data.yml b/omero/training-server/idr_data.yml deleted file mode 100644 index cc80eef0..00000000 --- a/omero/training-server/idr_data.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -- name: IDR data - hosts: ome-outreach - tasks: - - name: Clone study metadata - become: true - ansible.builtin.git: - dest: /uod/idr/metadata/{{ item.name }} - repo: https://github.com/IDR/{{ item.name }} - update: true - version: "{{ item.version }}" - loop: "{{ studies | default([]) }}" - - - name: Check existence of study data directory - ansible.builtin.stat: - path: /uod/idr/filesets/{{ item.name }} - register: stat_results - loop: "{{ studies | default([]) }}" - - - name: Fail if data directory is missing - ansible.builtin.fail: - msg: "/uod/idr/filesets/{{ item.item }} does not exist" - when: not item.stat.exists - loop: "{{ stat_results.results }}" diff --git a/omero/training-server/letsencrypt.yml b/omero/training-server/letsencrypt.yml deleted file mode 100644 index c9893927..00000000 --- a/omero/training-server/letsencrypt.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Additional Nginx configuration including Let's Encrypt -# Should be run when less than 30 days remain on the certificate - -- hosts: ome-outreach - - pre_tasks: - - - name: letsencrypt challenge directory - become: yes - file: - path: /srv/www/letsencrypt/.well-known/ - recurse: yes - state: directory - mode: 0755 - - roles: - - # Assume Nginx is already installed - # - role: ome.nginx - - # If we're not using letsencrypt assume this is CI so use a self-signed cert - - role: ome.ssl_certificate - when: not (https_letsencrypt_enabled | default(False)) - - # Lets encrypt with automatic renewal - # This will stop nginx when the certificate is first created - # For renewals we configure Nginx to serve the challenge - - role: ome.certbot - become: yes - certbot_create_if_missing: yes - certbot_admin_email: "{{ letsencrypt_email }}" - certbot_domains: - - "{{ https_certificate_domain }}" - certbot_create_standalone_stop_services: - - nginx - certbot_auto_renew_deploy_hooks: - - systemctl reload nginx - certbot_auto_renew_args: --webroot --webroot-path /srv/www/letsencrypt/ - # May be useful for testing: - #certbot_create_args: --test-cert - #certbot_auto_renew_args: --test-cert --force-renewal --webroot --webroot-path /srv/www/letsencrypt/ - # WARNING: If you have a test certificate and need to convert it to a - # real certificate you may need to run - # rm -rf /etc/letsencrypt/* - when: https_letsencrypt_enabled | default(False) - - tasks: - - - name: letsencrypt nginx configuration - become: yes - template: - src: templates/nginx-letsencrypt-conf.j2 - dest: /etc/nginx/conf.d-nested-includes/https.conf - mode: 0644 - notify: - - reload nginx - - handlers: - - name: reload nginx - become: yes - service: - name: nginx - state: reloaded - - vars: - # Must be defined somewhere - # https_certificate_domain: - letsencrypt_email: sysadmin@openmicroscopy.org - # This must match the expectations of certbot, do not change this: - https_letsencrypt_cert_path: "/etc/letsencrypt/live/{{ https_certificate_domain | default('localhost') }}" - # In production set this to True: - # https_letsencrypt_enabled: diff --git a/omero/training-server/maintenance/omero-restoredb.yml b/omero/training-server/maintenance/omero-restoredb.yml deleted file mode 100644 index 4455f3fa..00000000 --- a/omero/training-server/maintenance/omero-restoredb.yml +++ /dev/null @@ -1,81 +0,0 @@ ---- -# Restore the outreach OMERO database and data directory -# Requires: -# - data directory is in an uncompressed tar file -# - postgres has been dumped into a single file in custom format -# e.g. with omego db dump -# - ssh access to omero_restore_host from omero_restore_data_host -# for rsyncing files - -# This playbook will delete or overwrite existing data if necessary - -# Required runtime variables: -# - omero_restore_host: The target training server -# - omero_restore_data_host: The file server containing the archived repository - -# E.g. -# ansible-playbook omero-restoredb.yml --step -# -e omero_restore_host=ome-training-1.openmicroscopy.org -# -e omero_restore_data_host=example.openmicroscopy.org - - -- hosts: "{{ omero_restore_host }}" - - vars: - omero_restore_data_dump_file: /uod/idr/repos/outreach/outreach-omero-20190710/outreach-OMERO-20190710-1550.tar - omero_restore_db_dump_file: /uod/idr/repos/outreach/outreach-omero-20190710/omero-database-omero-20190710-154903-139204.pgdump - - tasks: - - # This may take a long time with no output - - name: Copy files - # Don't become root, this should ensure ssh forwarding works - synchronize: - src: "{{ item }}" - dest: "/tmp/{{ item | basename }}" - partial: true - delegate_to: "{{ omero_restore_data_host }}" - with_items: - - "{{ omero_restore_data_dump_file }}" - - "{{ omero_restore_db_dump_file }}" - - - name: Stop OMERO.server - become: true - service: - name: omero-server - state: stopped - - # Assumes the owner and group information in the archive is correct - - name: Restore OMERO data directory - become: true - unarchive: - src: "/tmp/{{ omero_restore_data_dump_file | basename }}" - dest: / - remote_src: true - - - name: Restore OMERO database - become: true - become_user: postgres - command: >- - pg_restore --clean -d omero - "/tmp/{{ omero_restore_db_dump_file | basename }}" - - - name: Upgrade OMERO database - become: omero-server - command: >- - /opt/omero/omego/bin/omego db upgrade - --serverdir /opt/omero/server/OMERO.server - - - name: Start OMERO.server - become: true - service: - name: omero-server - state: started - - - name: Delete temporary files - file: - path: "/tmp/{{ item | basename }}" - state: absent - with_items: - - "{{ omero_restore_data_dump_file }}" - - "{{ omero_restore_db_dump_file }}" diff --git a/omero/training-server/playbook.yml b/omero/training-server/playbook.yml deleted file mode 100644 index 353899bc..00000000 --- a/omero/training-server/playbook.yml +++ /dev/null @@ -1,511 +0,0 @@ ---- -# The OME training server(s) -# Set https_letsencrypt_enabled to True in production to ensure certbot -# handles the letsencrypt certificate setup - -- hosts: ome-outreach - pre_tasks: - - - name: Install Make Movie script Prerequisite | MEncoder - Repo - become: yes - yum: - name: http://li.nux.ro/download/nux/dextop/el7/x86_64/nux-dextop-release-0-5.el7.nux.noarch.rpm - state: present - - - name: OMERO.figure server-side prerequisites, script prerequisites + web server for decoupled OMERO.web - become: yes - yum: - name: "{{ item }}" - state: present - with_items: - - mencoder # For the 'make movie' script - - - name: Prerequisites for ldap - become: yes - yum: - name: "{{ item }}" - state: present - with_items: - - openldap-clients - - python-virtualenv - - gcc - - python-ldap - - # Since Nginx isn't installed until later the directories are created in advance - - name: Create nginx include directories - become: yes - file: - path: "{{ item }}" - state: directory - mode: 0755 - with_items: - - /etc/nginx/conf.d - - /etc/nginx/conf.d-nested-includes - - - name: NGINX - websocket proxy support - become: yes - template: - src: templates/nginx-confd-websockets-conf.j2 - dest: /etc/nginx/conf.d/websockets.conf - mode: 0644 - # Don't notify, nginx isn't installed yet - - roles: - - - role: ome.postgresql - postgresql_databases: - - name: omero - postgresql_users: - - user: omero - password: omero - databases: [omero] - - - role: ome.postgresql_backup - postgresql_backup_compress: true - postgresql_backup_dir: /OMERO/pgbackup - postgresql_backup_filename_format: "nightly-omero-%a.pgdump.gz" - postgresql_backup_minimum_expected_size: 100000000 - - - role: ome.versioncontrol_utils - - - role: ome.nfs_mount - - - role: ome.omero_server - omero_server_python_addons: - - "omero-cli-duplicate=={{ omero_cli_duplicate_release }}" - - "omero-cli-render=={{ omero_cli_render_release }}" - - "omero-metadata=={{ omero_metadata_release }}" - - "omero-py>={{ omero_py_release }}" - - "reportlab<3.6" - - markdown - - scipy - # For "simple frap with figure" script - - matplotlib - - - role: ome.omero_web - tags: ['web'] - idr_omero_web_public_url_filters_webclient_exclude: - - action - - annotate_(file|tags|comment|rating|map) - - script_ui - - ome_tiff - - figure_script - idr_omero_web_public_url_filters: - - api/ - - webadmin/myphoto/ - - mapr/ - - figure/ - - iviewer/ - - '$' - - gallery-api/ - - gallery_settings/ - - cell/ - - tissue/ - - webclient/(?!({{ idr_omero_web_public_url_filters_webclient_exclude | join('|') }})) - - webgateway/(?!(archived_files|download_as)) - omero_web_python_addons: - - "omero-py>={{ omero_py_release }}" - - omero_web_config_set: - omero.web.apps: - - "omero_iviewer" - - "omero_figure" - - "omero_fpbioimage" - - "omero_webtagging_autotag" - - "omero_webtagging_tagsearch" - - "omero_parade" - - "omero_mapr" - omero.web.ui.center_plugins: - - ["Auto Tag", "omero_webtagging_autotag/auto_tag_init.js.html", "auto_tag_panel"] - - ["Parade", "omero_parade/init.js.html", "omero_parade"] - omero.web.ui.top_links: - - ["Data", "webindex", {"title": "Browse Data via Projects, Tags etc"}] - - ["History", "history", {"title": "History"}] - - ["Help", "https://help.openmicroscopy.org/", {"title": "Open OMERO user guide in a new tab", "target": "new"}] - - ["Figure", "figure_index", {"title": "Open Figure in new tab", "target": "_blank"}] - - ["Tag Search", "tagsearch"] - - ["Genes", {"query_string": {"experimenter": -1}, "viewname": "maprindex_gene"}, {"title": "Find Gene annotations"}] - - ["Key-Value", {"viewname": "maprindex_keyvalue"}, {"title": "Search for manually-added Key-Value pairs"}] - omero.web.open_with: - - ["Image viewer", "webgateway", {"supported_objects": ["image"], "script_url": "webclient/javascript/ome.openwith_viewer.js"}] - - ["omero_figure", "new_figure", {"supported_objects":["images"], "target": "_blank", "label": "OMERO.figure"}] - - ["omero_fpbioimage", "fpbioimage_index", {"supported_objects":["image"], "script_url": "fpbioimage/openwith.js", "label": "FPBioimage"}] - - - omero_iviewer - - omero_iviewer_index - - supported_objects: - - images - - dataset - - well - script_url: omero_iviewer/openwith.js - label: OMERO.iviewer - - omero.web.viewer.view: omero_iviewer.views.index - omero.web.mapr.config: - - menu: gene - config: - default: - - "Gene Symbol" - all: - - "Gene Symbol" - - "Gene Identifier" - ns: - - "openmicroscopy.org/mapr/gene" - label: "Gene" - - menu: keyvalue - config: - default: - - "Any Value" - all: [] - ns: - - "openmicroscopy.org/omero/client/mapAnnotation" - label: "KeyValue" - omero.web.nginx_server_extra_config: - - 'include /etc/nginx/conf.d-nested-includes/*.conf;' - # TODO: Move the following into a shared variable - - > - location = /omero-ws { - proxy_pass https://127.0.0.1:4066; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - proxy_read_timeout 86400; - } - - omero.web.public.enabled: true - omero.web.public.password: "{{ omero_web_public_password_override | default('secret') }}" - omero.web.public.url_filter: "^/({{ idr_omero_web_public_url_filters | join('|') }})" - omero.web.public.user: "{{ omero_web_public_user_override | default('secret') }}" - - - role: ome.iptables_raw - - - role: ome.docker - docker_additional_options: - # Manually configure to avoid conflicts between Docker and system rules - iptables: false - - - role: ome.cli_utils - - tasks: - - name: Docker | python client - become: yes - yum: - name: docker-python - state: present - - - name: Create a figure scripts directory - become: yes - file: - path: /opt/omero/server/OMERO.server/lib/scripts/omero/figure_scripts - state: directory - mode: 0755 - recurse: yes - owner: "omero-server" - group: "omero-server" - - - name: Download the Figure_To_Pdf.py script - become: yes - get_url: - url: https://raw.githubusercontent.com/ome/omero-figure/v{{ omero_figure_release }}/omero_figure/scripts/omero/figure_scripts/Figure_To_Pdf.py - dest: /opt/omero/server/OMERO.server/lib/scripts/omero/figure_scripts/Figure_To_Pdf.py - mode: 0755 - owner: "omero-server" - group: "omero-server" - force: yes - - - name: Download the Dataset_Images_To_New_Figure.py script - become: yes - get_url: - url: https://raw.githubusercontent.com/ome/omero-guide-figure/f45f733a16852ae8b3c52ec93aef480d26b8e9f9/scripts/Dataset_Images_To_New_Figure.py - dest: /opt/omero/server/OMERO.server/lib/scripts/omero/figure_scripts/Dataset_Images_To_New_Figure.py - mode: 0755 - owner: "omero-server" - group: "omero-server" - force: yes - - - name: Download the Figure_Images_To_Dataset.py script - become: yes - get_url: - url: https://raw.githubusercontent.com/ome/omero-guide-figure/f45f733a16852ae8b3c52ec93aef480d26b8e9f9/scripts/Figure_Images_To_Dataset.py - dest: /opt/omero/server/OMERO.server/lib/scripts/omero/figure_scripts/Figure_Images_To_Dataset.py - mode: 0755 - owner: "omero-server" - group: "omero-server" - force: yes - - - name: Create a workshop_scripts directory - become: yes - file: - path: /opt/omero/server/OMERO.server/lib/scripts/omero/workshop_scripts - state: directory - mode: 0755 - recurse: yes - owner: "omero-server" - group: "omero-server" - - - name: Download the Scipy_Gaussian_Filter.py script - become: yes - get_url: - url: https://raw.githubusercontent.com/ome/omero-guide-python/v{{ ome_training_scripts_release }}/scripts/scipy_gaussian_filter.py - dest: /opt/omero/server/OMERO.server/lib/scripts/omero/workshop_scripts/Scipy_Gaussian_Filter.py - mode: 0755 - owner: "omero-server" - group: "omero-server" - force: yes - - - name: Download the simple_frap.py script - become: yes - get_url: - url: https://raw.githubusercontent.com/ome/omero-guide-python/v{{ ome_training_scripts_release }}/scripts/simple_frap.py - dest: /opt/omero/server/OMERO.server/lib/scripts/omero/workshop_scripts/simple_frap.py - mode: 0755 - owner: "omero-server" - group: "omero-server" - force: yes - - - name: Download the simple_frap_with_figure.py script - become: yes - get_url: - url: https://raw.githubusercontent.com/ome/omero-guide-python/v{{ ome_training_scripts_release }}/scripts/simple_frap_with_figure.py - dest: /opt/omero/server/OMERO.server/lib/scripts/omero/workshop_scripts/simple_frap_with_figure.py - mode: 0755 - owner: "omero-server" - group: "omero-server" - force: yes - - - name: Create a directory for ldap scripts - become: yes - file: - path: /home/ldap - state: directory - mode: 0755 - recurse: yes - - - name: Download the ldap scripts - become: yes - get_url: - url: https://raw.githubusercontent.com/openmicroscopy/apacheds-docker/{{ apache_docker_release }}/bin/ldapmanager - dest: /home/ldap/ldapmanager - mode: 0755 - force: yes - - - name: Add DropBox folder for trainer-1 - become: yes - file: - path: /home/DropBox/trainer-1 - state: directory - mode: 0755 - recurse: yes - owner: "omero-server" - group: "omero-server" - - - name: Add operating system user "importer1" - become: true - user: - name: "importer1" - state: present - groups: "{{ omero_server_system_managedrepo_group }}" - password: "{{ os_system_users_password | password_hash('sha512', 'ome') }}" - - - name: Allow managed repo group to login - become: yes - lineinfile: - path: /etc/security/access.conf - regexp: "{{ omero_server_system_managedrepo_group }}" - insertbefore: BOF - line: "+:{{ omero_server_system_managedrepo_group }}:ALL" - - - name: Run docker for ldap - become: yes - docker_container: - image: openmicroscopy/apacheds:{{ apache_docker_release }} - name: ldap - published_ports: - - "10389:10389" - state: started - restart_policy: always - - - name: Run docker for omero-ms-zarr - become: yes - docker_container: - image: openmicroscopy/omero-ms-zarr:{{ omero_ms_zarr_release }} - name: omero_ms_zarr - env: - CONFIG_omero_db_host: localhost - CONFIG_omero_db_user: omero - CONFIG_omero_db_pass: omero - CONFIG_omero_db_name: omero - CONFIG_omero_data_dir: /OMERO - network_mode: host - restart_policy: always - state: started - pull: yes - volumes: - - "/OMERO:/OMERO:ro" - - - name: Create minio config directory - become: yes - file: - path: /etc/minio - state: directory - mode: 0755 - - - name: Check if minio admin credentials exists - become: yes - stat: - path: /etc/minio/docker-minio.env - register: _minio_docker_env_st - - - name: Create random minio admin credentials file - become: yes - copy: - content: | - MINIO_ACCESS_KEY={{ lookup('password', '/dev/null length=12') }} - MINIO_SECRET_KEY={{ lookup('password', '/dev/null length=24') }} - dest: /etc/minio/docker-minio.env - mode: 0644 - when: not _minio_docker_env_st.stat.exists - - - name: Run docker for minio - become: yes - docker_container: - image: minio/minio:{{ minio_docker_release }} - name: minio - command: server /srv/minio - env_file: /etc/minio/docker-minio.env - published_ports: - - "9000:9000" - state: started - restart_policy: always - volumes: - - "/srv/minio:/srv/minio" - - - name: Nginx - docker webservices support - become: yes - template: - src: templates/nginx-confdnestedincludes-{{ item }}.j2 - dest: /etc/nginx/conf.d-nested-includes/{{ item }}.conf - mode: 0644 - with_items: - - omero-ms-zarr - - minio-publicscratch - notify: - - restart nginx - - # https://fralef.me/docker-and-iptables.html - # https://blog.daknob.net/debian-firewall-docker/ - # Allow: - # - all outbound from Docker containers - # - incoming from host localhost - - - name: Iptables Docker forward rules - become: yes - iptables_raw_25: - name: docker_outreach_rules - rules: | - -A FORWARD -i docker0 -o {{ external_nic }} -j ACCEPT - -A FORWARD -i {{ external_nic }} -o docker0 -j ACCEPT - state: present - - - name: Iptables Docker nat rules - become: yes - iptables_raw_25: - name: docker_outreach_nat - table: nat - rules: | - -A POSTROUTING -o {{ external_nic }} -j MASQUERADE - # Allow world to access 10389? - -A INPUT -p tcp -m tcp --dport 10389 -s 0.0.0.0/0 -j ACCEPT - state: present - - # TODO: Move to an independent role, currently bundled in - # https://github.com/manics/ansible-role-minio-s3-gateway/blob/0.1.0/tasks/minio-client.yml - - name: Download minio client - become: true - get_url: - url: - https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2020-11-25T23-04-07Z - checksum: - sha256:985c43f9bec8fdc4ef2ee44c63c9657e10c4cfeb5cb949179d6d693f7428c314 - dest: /usr/local/bin/mc - mode: u=rwx,g=rx,o=rx - - # Crypted passwords generated using - # https://docs.ansible.com/ansible/latest/reference_appendices/faq.html#how-do-i-generate-crypted-passwords-for-the-user-module - vars: - #omero_server_datadir_chown: True - #temporal upgrade force for omero server workaround - #omero_server_checkupgrade_comparator: '!=' - postgresql_version: "13" - omero_server_selfsigned_certificates: True - omero_server_system_managedrepo_group: managed_repo_group - omero_server_datadir_managedrepo_mode: u=rwX,g=srwX,o=rX,+t - omero_server_datadir_chown: False - omero_server_release: "{{ omero_server_release_override | default('5.6.8') }}" - omero_web_release: "{{ omero_web_release_override | default('5.22.1') }}" - omero_figure_release: "{{ omero_figure_release_override | default('6.0.1') }}" - omero_fpbioimage_release: "{{ omero_fpbioimage_release_override | default('0.4.1') }}" - omero_iviewer_release: "{{ omero_iviewer_release_override | default('0.13.0') }}" - omero_mapr_release: "{{ omero_mapr_release_override | default('0.5.0') }}" - omero_parade_release: "{{ omero_parade_release_override | default('0.2.4') }}" - omero_py_release: "{{ omero_py_release_override | default('5.15.0') }}" - - # The omero_web_apps_* vars are used by the ome.omero_web role under - # Python 3 otherwise ignored - omero_web_apps_packages: - - "omero-figure=={{ omero_figure_release }}" - - "omero-fpbioimage=={{ omero_fpbioimage_release }}" - - "omero-iviewer=={{ omero_iviewer_release }}" - - "omero-mapr=={{ omero_mapr_release }}" - - "omero-parade=={{ omero_parade_release }}" - - "omero-webtagging-autotag=={{ omero_webtagging_autotag_release }}" - - "omero-webtagging-tagsearch=={{ omero_webtagging_tagsearch_release }}" - - ome_training_scripts_release: "{{ ome_training_scripts_release_override | default('0.2.0') }}" - omero_webtagging_autotag_release: "{{ omero_webtagging_autotag_release_override | default('3.2.0') }}" - omero_webtagging_tagsearch_release: "{{ omero_webtagging_tagsearch_release_override | default('3.2.0') }}" - omero_cli_duplicate_release: "{{ omero_cli_duplicate_release_override | default('0.4.0') }}" - omero_metadata_release: "{{ omero_metadata_release_overrride | default('0.8.0') }}" - omero_cli_render_release: "{{ omero_cli_render_release_override | default('0.7.0') }}" - os_system_users_password: "{{ os_system_users_password_override | default('ome') }}" - apache_docker_release: "{{ apache_docker_release_override | default('0.6.0') }}" - omero_ms_zarr_release: "{{ omero_ms_zarr_release_override | default('latest') }}" - minio_docker_release: "{{ minio_docker_release_override | default('RELEASE.2020-11-25T22-36-25Z') }}" - ldap_password: "{{ ldap_password_override | default ('secret') }}" - omero_server_config_set: - #omero.fs.importUsers: "fm1" - omero.certificates.owner: "/C=UK/ST=Scotland/L=Dundee/O=OME" - omero.client.icetransports: ssl,wss,tcp - omero.fs.watchDir: "/home/DropBox" - omero.fs.importArgs: "-T \"regex:^.*/(?.*?)\"" - omero.db.poolsize: 60 - omero.jvmcfg.percent.blitz: 50 - omero.jvmcfg.percent.indexer: 20 - omero.jvmcfg.percent.pixeldata: 20 - omero.glacier2.IceSSL.Ciphers: "ADH:HIGH" - omero.glacier2.IceSSL.DefaultDir: /opt/omero/server/selfsigned - omero.glacier2.IceSSL.CAs: server.pem - omero.glacier2.IceSSL.CertFile: server.p12 - # This password doesn't need to be secret - omero.glacier2.IceSSL.Password: secret - omero.fs.repo.path: "%user%_%userId%/%thread%//%year%-%month%/%day%/%time%" - omero.ldap.config: "true" - omero.ldap.urls: "ldap://localhost:10389" - omero.ldap.base: "dc=openmicroscopy,dc=org" - omero.ldap.group_filter: "(objectClass=groupOfUniqueNames)" - omero.ldap.group_mapping: "name=cn" - omero.ldap.new_user_group: "MyData" - omero.ldap.new_user_group_owner: "(owner=@{dn})" - omero.ldap.password: "{{ ldap_password }}" - omero.ldap.sync_on_login: "true" - omero.ldap.user_filter: "(objectClass=person)" - omero.ldap.user_mapping: "omeName=uid,firstName=givenName,lastName=sn,email=mail" - omero.ldap.username: "uid=admin,ou=system" - omero.server.nodedescriptors: "master:Blitz-0,Indexer-0,Processor-0,Storm,Tables-0" - - external_nic: "{{ ansible_default_ipv4.interface }}" - -- name: Import letsencrypt - import_playbook: letsencrypt.yml - -- name: Import idr data - import_playbook: idr_data.yml diff --git a/omero/training-server/templates b/omero/training-server/templates deleted file mode 120000 index 564a409d..00000000 --- a/omero/training-server/templates +++ /dev/null @@ -1 +0,0 @@ -../templates \ No newline at end of file diff --git a/postgres/ome-pg-prod.yml b/postgres/ome-pg-prod.yml deleted file mode 100644 index be1e9109..00000000 --- a/postgres/ome-pg-prod.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -- hosts: ome-pg-prod1.openmicroscopy.org - roles: - - - role: ome.postgresql - postgresql_version: "9.6" - postgresql_server_listen: "'*'" - postgresql_databases: - - name: idr-redmine - owner: idr-redmine - restrict: True - postgresql_users: - - user: idr-redmine - password: "{{ idr_redmine_postgres_password | default('idr-redmine') }}" - databases: - - idr-redmine - postgresql_server_auth: - - database: idr-redmine - user: idr-redmine - address: "{{ idr_redmine_postgres_auth_ip | default('0.0.0.0/0') }}" - - database: idr-redmine - user: idr-redmine - address: "{{ idr_redmine_postgres_auth_ip2 | default('0.0.0.0/0') }}" - - - role: ome.nfs_mount - # Parameters for this role are internal - - - role: ome.postgresql_backup - postgresql_backup_dir: /mnt/backups/ - postgresql_backup_filename_format: "{{ ansible_hostname }}-%a.pgdump" - postgresql_backup_minimum_expected_size: 100000 diff --git a/release/group_vars/all.yml b/release/group_vars/all.yml deleted file mode 100644 index 8763f782..00000000 --- a/release/group_vars/all.yml +++ /dev/null @@ -1,67 +0,0 @@ ---- -www_folders: - - /uod/idr/www/docs.openmicroscopy.org - - /uod/idr/www/downloads.openmicroscopy.org -# Public image format folders -# All keys of this dictionary are expected to be a valid format folder and -# have a subfolder called public containing the public sample images -public_folders: - amira: 'AmiraMesh' - bdv: 'BDV' - becker-hickl-spc: 'SPC-FIFO' - cellomics: 'Cellomics' - cellh5: 'CellH5' - cellworx: 'CellWorX' - columbus: 'PerkinElmer-Columbus' - deltavision: 'DV' - dicom: 'DICOM' - ecat7: 'ECAT7' - flex: 'Flex' - fv1000: 'Olympus-FluoView' - gatan: 'Gatan' - hamamatsu: 'Hamamatsu-NDPI' - hamamatsu-vms: 'Hamamatsu-VMS' - ics: 'ICS' - incell3000: 'InCell3000' - incell: 'InCell2000' - imaris: 'Imaris-IMS' - klb: 'KLB' - leica-lif: 'Leica-LIF' - leica-scn: 'Leica-SCN' - leica-xlef-lof: 'Leica-XLEF' - leo: 'LEO' - metaxpress: 'MetaXpress' - micromanager: 'Micro-Manager' - mrc: 'MRC' - nd2: 'ND2' - nifti: 'NIfTI' - nrrd: 'NRRD' - obf: 'OBF' - olympus-oir: 'Olympus-OIR' - ome-xml: 'OME-XML' - ome-tiff: 'OME-TIFF' - perkinelmer-operetta: 'PerkinElmer-Operetta' - png: 'PNG' - svs: 'SVS' - tiff: 'TIFF' - trestle: 'Trestle' - ventana: 'Ventana' - vectra-qptiff: 'Vectra-QPTIFF' - zeiss-czi: 'Zeiss-CZI' -# List containing special public images/folders that do not meet the standard -# layout above -special_public_folders: - - src: '../../../../repos/curated/samples/carlos/big.tiff' - dest: 'gateway_tests/big.tiff' - - src: '../../../../repos/curated/samples/ome/CHOBI_d3d.dv' - dest: 'gateway_tests/CHOBI_d3d.dv' - - src: '../../../../repos/curated/samples/ome/tinyTest.d3d.dv' - dest: 'gateway_tests/tinyTest.d3d.dv' - - src: '../../../../repos/curated/cellomics/public/' - dest: 'HCS/BBBC' - - src: '../../../../repos/curated/incell/public/' - dest: 'HCS/INCELL2000' - - src: '../../../../repos/curated/perkinelmer-operetta/public/' - dest: 'HCS/Operetta' - - src: '../../../repos/curated/zip/u-track/' - dest: 'u-track' diff --git a/release/permissions.yml b/release/permissions.yml deleted file mode 100644 index 400bddfe..00000000 --- a/release/permissions.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: Permissions - hosts: idr0-slot3.openmicroscopy.org - become: true - tasks: - - name: Set perms /uod/idr/www - ansible.builtin.file: - path: /uod/idr/www - state: directory - owner: root - group: root - mode: 0755 - - - name: Set perms2 - ansible.builtin.file: - path: "{{ item }}" - state: directory - owner: root - group: root - mode: 0755 - with_items: "{{ www_folders }}" - - - name: Set perms3 www_folders - ansible.builtin.find: - paths: "{{ www_folders }}" - file_type: "directory" - recurse: "no" - register: "products" - - - name: Set perms3 - ansible.builtin.file: - path: "{{ item }}" - state: directory - owner: root - group: lsd - mode: 01775 - with_items: "{{ products.files | map(attribute='path') | list }}" diff --git a/release/presentations.yml b/release/presentations.yml deleted file mode 100644 index f953a14a..00000000 --- a/release/presentations.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: Presentations playbook - hosts: idr0-slot3.openmicroscopy.org - become: true -# yamllint disable rule:latest - tasks: - - name: Update presentations - ansible.builtin.git: - repo: https://github.com/ome/presentations - force: false - update: true - dest: "/uod/idr/www/downloads.openmicroscopy.org/presentations" -# yamllint disable rule:latest diff --git a/release/public-images.yml b/release/public-images.yml deleted file mode 100644 index ce649a69..00000000 --- a/release/public-images.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -- name: Public images - hosts: idr0-slot3.openmicroscopy.org - become: true - tasks: - - name: Create symlinks for public images - ansible.builtin.file: - force: true - src: "../../../repos/curated/{{ item.key }}/public/" - dest: > - "/uod/idr/www/downloads. - openmicroscopy.org/ - images/{{ item.value }}" - state: link - with_dict: "{{ public_folders }}" - - name: Check public images - ansible.builtin.file: - force: true - src: "{{ item.src }}" - dest: > - "/uod/idr/www/downloads. - openmicroscopy.org/ - images/{{ item.dest }}" - state: link - with_items: "{{ special_public_folders }}" diff --git a/release/release-acceptance.yml b/release/release-acceptance.yml deleted file mode 100644 index 75adaad6..00000000 --- a/release/release-acceptance.yml +++ /dev/null @@ -1,97 +0,0 @@ ---- -- name: Release acceptance - hosts: idr0-slot3.openmicroscopy.org - become: true - tasks: - - name: Check mandatory variables are defined - fail: - msg: Please pass 'product' and 'version' variables - when: product is not defined and version is not defined - - - name: Check the release component exist - stat: - path: "{{ item }}/{{ product }}/{{ version }}/" - register: s - with_items: "{{ www_folders }}" - - - name: Check the release component exist - fail: - msg: "{{ item }} does not exist" - with_items: "{{ s.results }}" - when: item.stat is not defined or not item.stat.exists - - - name: Check pre-release - set_fact: - prerelease: "{{ '-' in version }}" - - - name: Define minor and major versions - set_fact: - minorversion: "{{ version.split('.')[:2] | join('.') }}" - majorversion: "{{ version.split('.')[:1] | join('.') }}" - - - name: Remove .htaccess file - file: - path: "{{ item }}/{{ product }}/{{ version }}/.htaccess" - state: absent - with_items: "{{ www_folders }}" - - - name: Make release folders read-only and owned by root - file: - path: "{{ item }}/{{ product }}/{{ version }}" - state: directory - owner: root - group: root - recurse: true - mode: 01555 - with_items: "{{ www_folders }}" - - - name: Create minor version directory - file: - path: "{{ item }}/{{ product }}/{{ minorversion }}" - state: directory - mode: 0755 - with_items: "{{ www_folders }}" - when: not prerelease - - - name: Create minor version redirects - copy: - dest: "{{ item }}/{{ product }}/{{ minorversion }}/.htaccess" - content: "Redirect 301 /{{ product }}/{{ minorversion }} \ - /{{ product }}/{{ version }}" - mode: 0644 - with_items: "{{ www_folders }}" - when: not prerelease - - - name: Create minor version directory - file: - path: "{{ item }}/{{ product }}/{{ majorversion }}" - state: directory - mode: 0755 - with_items: "{{ www_folders }}" - when: not prerelease - - - name: Create major version redirects - copy: - dest: "{{ item }}/{{ product }}/{{ majorversion }}/.htaccess" - content: "Redirect 301 /{{ product }}/{{ majorversion }} \ - /{{ product }}/{{ version }}" - mode: 0644 - with_items: "{{ www_folders }}" - when: not prerelease - - - name: Create latest version directory - file: - path: "{{ item }}/{{ product }}/latest" - state: directory - mode: 0755 - with_items: "{{ www_folders }}" - when: not prerelease - - - name: Create latest version redirect - copy: - dest: "{{ item }}/{{ product }}/latest/.htaccess" - content: "Redirect 301 /{{ product }}/latest \ - /{{ product }}/{{ version }}" - mode: 0644 - with_items: "{{ www_folders }}" - when: not prerelease diff --git a/requirements.yml b/requirements.yml index a1759a69..248a7e23 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,107 +1,24 @@ --- - -- name: ome.certbot - src: https://github.com/ome/ansible-role-certbot/archive/0.1.0.tar.gz - version: 0.1.0 - -- src: ome.cli_utils - version: 1.1.1 - -- src: ome.deploy_archive - version: 0.1.4 - -- src: ome.docker - version: 3.1.1 - -- src: ome.ice - version: 4.3.0 - -- src: ome.java - version: 2.1.0 - -- name: ome.iptables_raw - version: 0.3.1 - -- src: ome.lvm_partition - version: 1.1.1 - -- name: ome.network - version: 1.1.4 - -- src: ome.nginx - version: 2.1.1 - -- name: ome.nginx_proxy - version: 1.15.2 - -- src: ome.nfs_mount - version: 1.3.0 - - src: ome.omero_common - version: 0.3.4 - - src: ome.basedeps - version: 1.2.0 - -- name: ome.omero_prometheus_exporter - version: 0.3.6 - -- name: ome.omero_server - version: 4.0.2 - -- src: ome.omero_user - version: 0.3.0 - -- name: ome.omero_web - version: 4.0.1 - +- src: ome.java - src: ome.python3_virtualenv - version: 0.1.2 - -- src: ome.omero_web_django_prometheus - version: 0.3.0 - +- src: ome.ice - src: ome.postgresql - version: 5.2.0 - -- src: ome.postgresql_backup - version: 0.2.1 - - src: ome.postgresql_client - version: 0.2.0 - -- src: ome.prometheus - version: 0.5.1 - -- src: ome.prometheus_jmx - version: 0.2.2 - -- src: ome.prometheus_node - version: 0.2.2 - -- src: ome.prometheus_postgres - version: 0.4.2 - +- src: ome.deploy_archive +- src: ome.omero_server +- src: ome.omero_web +- src: ome.nginx - src: ome.redis - version: 1.1.1 - -- name: ome.selinux_utils - version: 1.0.3 - -- src: ome.ssl_certificate - version: 0.4.0 - -- src: ome.sudoers - version: 1.0.4 - -- src: ome.system_monitor_agent - version: 0.1.1 - -- src: ome.upgrade_distpackages - version: 1.1.3 - +- src: ome.selinux_utils - src: ome.versioncontrol_utils - version: 1.0.2 - -- src: idr.redmine_tracker - version: 0.1.0 +- src: ome.ssl_certificate +- src: nfs_mount +- src: nfs_share +- src: iptables_raw +- src: ome.cli_utils +- src: ome.docker +- src: ome.postgresql_backup +- src: ome.omero_user +- src: ome.lvm_partition diff --git a/site.yml b/site.yml deleted file mode 100644 index 10bf0d04..00000000 --- a/site.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -# ansible-playbook -i /prod-hosts site.yml - -# For new hosts you may also need to run the bootstrap playbook to setup -# networking and initial partitions: -# - import_playbook: bootstrap/playbook.yml - -# deployment of two OMERO.web instances for hosting Dundee's production OMERO. -# One of the two instances is for publication data ("ns-web-pub"). -- import_playbook: omero/nightshade-webclients.yml - -# nightshade.openmicroscopy.org OMERO.server -- import_playbook: omero/ome-dundeeomero.yml - -# https://demo.openmicroscopy.org OMERO.server and OMERO.web -- import_playbook: omero/ome-demoserver.yml - -# Firewall configuration for all UoD OMERO servers -# The training-server playbooks modify the firewall configuration so this must -# be run first to setup the basic rules -- import_playbook: omero/omero-firewall.yml - -# https://outreach.openmicroscopy.org -# https://workshop.openmicroscopy.org -- import_playbook: omero/training-server/playbook.yml - -# Internal monitoring configuration for all UoD OMERO servers -- import_playbook: omero/omero-monitoring-agents.yml - -# https://www.openmicroscopy.org/ -- import_playbook: www/playbook.yml - -# OME proxied service -- import_playbook: web-proxy/playbook.yml diff --git a/vendor/.gitignore b/vendor/.gitignore deleted file mode 100644 index 72e8ffc0..00000000 --- a/vendor/.gitignore +++ /dev/null @@ -1 +0,0 @@ -* diff --git a/web-proxy/playbook.yml b/web-proxy/playbook.yml deleted file mode 100644 index 84a0fe8a..00000000 --- a/web-proxy/playbook.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -# Playbook for maintaining OME production web proxies - -- name: Web proxy - hosts: web-proxies - roles: - - role: ome.network - tags: network - - role: ome.lvm_partition - tags: lvm - lvm_lvname: root - lvm_lvmount: / - lvm_lvsize: "{{ root_size }}" - lvm_lvfilesystem: "{{ root_filesystem }}" - - role: ome.lvm_partition - tags: lvm - lvm_lvname: var_log - lvm_lvmount: /var/log - lvm_lvsize: "{{ varlog_size }}" - lvm_lvfilesystem: "{{ root_filesystem }}" - - role: ome.ssl_certificate - - role: ome.nginx_proxy - - handlers: - - name: Reload nginx - listen: ssl certificate changed - become: true - ansible.builtin.service: - name: nginx - state: reloaded - - vars: - nginx_version: 1.18.0 diff --git a/www/README.txt b/www/README.txt deleted file mode 100644 index 8cf4801d..00000000 --- a/www/README.txt +++ /dev/null @@ -1,7 +0,0 @@ -### ansible playbook & requirements for www server - -- after installing ansible and ansible-galaxy, - ansible-galaxy install -r ../requirements.yml -p roles - -- install and configure server - ansible-playbook playbook.yml diff --git a/www/Vagrantfile b/www/Vagrantfile deleted file mode 100644 index fec73b26..00000000 --- a/www/Vagrantfile +++ /dev/null @@ -1,14 +0,0 @@ -Vagrant.configure("2") do |config| - config.vm.box = "centos/7" - config.vm.provider "virtualbox" do |vb| - config.vm.network "forwarded_port", guest: 80, host: 8080 - config.vm.network "forwarded_port", guest: 443, host: 8443 - vb.customize ["modifyvm", :id, "--memory", "2048"] - end - - config.vm.provision "ansible" do |ansible| - ansible.playbook = "playbook.yml" - ansible.galaxy_role_file = "requirements.yml" - ansible.skip_tags = "lvm" - end -end diff --git a/www/files/deploy b/www/files/deploy deleted file mode 100644 index 2ec6aef4..00000000 --- a/www/files/deploy +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -import argparse -import json -import os -import sys -import tarfile -# Python 2 and 3 compatible so this can be run on RHEL7 without a virtualenv -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen - - -def main(version, parent, dry_run): - # https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#releases - # No paging, assume no-one will install a really old version - r = urlopen('https://api.github.com/repos/ome/www.openmicroscopy.org/releases') - assert r.code == 200 - releases = json.load(r) - - if version == 'latest': - release = releases[0] - else: - release = None - for check in releases: - if check['tag_name'] == version: - release = check - break - if release is None: - print('Failed to find release {}'.format(version)) - sys.exit(1) - - tag = release['tag_name'] - - dst = os.path.join(parent, tag) - sym = os.path.join(parent, 'html') - - if os.path.exists(dst): - print('{} already exists, not downloading'.format(dst)) - elif dry_run: - print('Would download {}'.format(dst)) - else: - www_assets = [a for a in release['assets'] if a['name'] == 'www.openmicroscopy.org.tar.gz'] - assert len(www_assets) == 1, 'Expected one asset named www.openmicroscopy.org.tar.gz' - url = www_assets[0]['browser_download_url'] - - h = urlopen(url) - thetarfile = tarfile.open(fileobj=h, mode="r|gz") - thetarfile.extractall(path=dst) - h.close() - print('Extracted {} to {}'.format(url, dst)) - - if os.path.exists(sym): - assert os.path.islink(sym), '{} is not a symlink'.format(sym) - target = os.readlink(sym) - if target == dst: - print('{} already points to {}, no changes made'.format(dst, sym)) - sys.exit(0) - elif dry_run: - print('Would remove symlink {} (target={})'.format(sym, target)) - else: - print(target) - # Mutator - os.remove(sym) - if dry_run: - print('Would symlink {} to {}'.format(dst, sym)) - else: - # Mutator - os.symlink(dst, sym) - print('Symlinked {} to {}'.format(dst, sym)) - sys.exit(1) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - xor = parser.add_mutually_exclusive_group() - xor.add_argument('-n','--dry-run', action='store_true', default=True) - xor.add_argument('-f','--force', action='store_false', dest="dry_run") - parser.add_argument( - '--parentdir', default='/var/www/www.openmicroscopy.org', - help='Web-server directory for www.openmicroscopy.org') - parser.add_argument('--version', default='latest', - help='Release to download') - args = parser.parse_args() - main(args.version, args.parentdir, args.dry_run) diff --git a/www/playbook.yml b/www/playbook.yml deleted file mode 100644 index 8c8fb53b..00000000 --- a/www/playbook.yml +++ /dev/null @@ -1,116 +0,0 @@ -# Install NGINX, and prepare the OME (UoD/SLS) prerequisites - -- name: Playbook www - hosts: www - environment: - PATH: /usr/local/bin:{{ ansible_env.PATH }} - pre_tasks: - - name: Install open-vm-tools if system is a VMware vm - become: true - ansible.builtin.yum: - name: open-vm-tools - state: installed - when: > - ((ansible_virtualization_type is defined) - and (ansible_virtualization_type == "VMware")) - - # Two tasks here which could instead use the role at - # https://github.com/openmicroscopy/ansible-role-lvm-partition/, - # but that role - # will change the device name to symlinks rather than - # existing device ID. - - name: storage | Resize root LV - tags: lvm - become: true - lvol: - lv: root - vg: "{{ lvm_vgname }}" - size: "{{ provision_root_lvsize }}" - shrink: false - - # Grow the filesystem to fill the LV - - name: storage | Resize root FS - tags: lvm - become: true - filesystem: - fstype: "{{ filesystem }}" - dev: /dev/mapper/{{ lvm_vgname }}-root - resizefs: true - - - name: storage | Resize var_log LV - tags: lvm - become: true - lvol: - lv: var_log - vg: "{{ lvm_vgname }}" - size: "{{ provision_varlog_lvsize }}" - shrink: false - - # Grow the filesystem to fill the LV - - name: storage | Resize var_log FS - tags: lvm - become: true - filesystem: - fstype: "{{ filesystem }}" - dev: /dev/mapper/{{ lvm_vgname }}-var_log - resizefs: true - - roles: - # Now OME are using RHEL without Spacewalk, - # the current best-method of - # checking `is server deployed in Dundee/SLS` - # is checking for the SLS nameservers. - - role: ome.system_monitor_agent - tags: monitoring - when: "'10.1.255.216' in ansible_dns.nameservers" - - - role: ome.sudoers - sudoers_individual_commands: - - user: "%omedev" - become: ALL - command: "NOPASSWD: /usr/local/bin/deploy *" - - post_tasks: - - - name: Check_MK logwatch plugin | check for plugin existence - tags: monitoring - ansible.builtin.stat: - path: "{{ check_mk_agent_plugin_path }}/mk_logwatch" - register: check_mk_logwatch_plugin_st - - - name: Check_MK logwatch plugin | activate the plugin - tags: monitoring - become: true - command: >- - cp "{{ check_mk_agent_plugin_path }}/mk_logwatch" - /usr/share/check-mk-agent/plugins/ - creates=/usr/share/check-mk-agent/plugins/mk_logwatch - when: check_mk_logwatch_plugin_st.stat.exists - - - name: Check_MK logwatch plugin | check for default config file - tags: monitoring - ansible.builtin.stat: - path: "{{ check_mk_agent_config_example_path }}/logwatch.cfg" - register: check_mk_logwatch_plugin_conf_st - - - name: Check_MK logwatch plugin | copy the default config - tags: monitoring - become: true - command: >- - cp "{{ check_mk_agent_config_example_path }}/logwatch.cfg" - "{{ check_mk_agent_config_path }}/logwatch.cfg" - creates="{{ check_mk_agent_config_path }}/logwatch.cfg" - when: check_mk_logwatch_plugin_conf_st.stat.exists - - vars: - # Check_MK (system monitoring) paths - check_mk_agent_plugin_path: /usr/share/check-mk-agent/available-plugins - check_mk_agent_config_example_path: "/usr/share\ - /check_mk/agents/cfg_examples" - check_mk_agent_config_path: /etc/check-mk-agent - - filesystem: "ext4" - -- name: Import www deploy - import_playbook: www-deploy.yml -# www-deploy.yml includes www-jekyll.yml diff --git a/www/tests/check_redirects.py b/www/tests/check_redirects.py deleted file mode 100644 index 2e45aa1d..00000000 --- a/www/tests/check_redirects.py +++ /dev/null @@ -1,179 +0,0 @@ -# Test redirects -# -# Test the default host: -# pytest check_redirects.py -# -# Test a different host: -# HOST=http://www-dev.openmicroscopy.org pytest check_redirects.py - -import os -import pytest -import requests - -HOST_OME = os.getenv('HOST', 'https://ome-www.openmicroscopy.org') -HOST_OPENMICROSCOPY = os.getenv('HOST', 'https://www.openmicroscopy.org') -hosts = (HOST_OME, HOST_OPENMICROSCOPY) -suffixes = ['', '/'] -redirect_uris = [ - ('/site', '/'), - ('/site/about', '/about'), - ('/site/about/licensing', '/licensing'), - ('/site/about/licensing-attribution', '/licensing'), - ('/site/about/licensing-attribution/licensing', '/licensing'), - ('/site/about/ome-contributors', '/contributors'), - ('/site/about/partners', '/commercial-partners'), - ('/site/about/development-teams', '/teams'), - ('/site/about/publications', '/citing-ome'), - ('/site/about/who-ome', '/teams'), - ('/site/about/what-omero/overview', '/omero'), - ('/site/about/roadmap', '/about'), - ('/site/about/project-history', '/about'), - - ('/site/community', '/support'), - ('/site/community/mailing-lists', '/support'), - ('/site/events', '/events'), - ('/site/community/minutes/conference-calls', '/on-the-web'), - ('/site/community/minutes/meetings/12th-annual-users-meeting-2017', - '/events/12th-annual-users-meeting-2017.html'), - ('/site/community/minutes/meetings/11th-annual-users-meeting-2016', - '/events/11th-annual-users-meeting-2016.html'), - ('/site/community/minutes/meetings/10th-annual-users-meeting-june-2015', '/events/10th-annual-users-meeting-june-2015.html'),# noqa - ('/site/community/minutes/meetings/9th-annual-users-meeting-june-2014', '/events/9th-annual-users-meeting-june-2014.html'),# noqa - ('/site/community/jobs', '/careers'), - - ('/site/products', '/products'), - ('/site/products/bio-formats', '/bio-formats'), - ('/site/products/bio-formats/downloads', '/bio-formats/downloads/'), - ('/site/products/omero', '/omero'), - ('/site/products/omero/downloads', '/omero/downloads/'), - ('/site/products/omero/feature-list', '/omero/features/'), - ('/site/products/omero/secvuln', '/security/advisories/'), - ('/site/products/ome5/secvuln', '/security/advisories/'), - ('/site/products/omero/secvuln/2014-SV3-csrf', - '/security/advisories/2014-SV3-csrf/'), - - ('/site/support', '/docs'), - ('/site/support/ome-artwork', '/artwork'), - ('/site/support/ome-artwork/artwork-usage', '/artwork'), - ('/site/news', '/announcements'), - - ('/info/vulnerabilities', '/security/advisories/'), - ('/info/vulnerabilities/2014-SV3-csrf', - '/security/advisories/2014-SV3-csrf/'), - ('/info/omero', '/omero'), - ('/info/cls', '/omero/downloads/'), - ('/info/download', '/omero/downloads/'), - ('/info/downloads', '/omero/downloads/'), - ('/info/attribution', '/licensing/'), -] -external_uris = [ - ('/omero-blog', 'http://blog.openmicroscopy.org'), - ('/site/about/development-teams/glencoe-software', 'https://www.glencoesoftware.com/team.html'),# noqa - ('/site/community/scripts', 'https://docs.openmicroscopy.org/latest/omero/developers/scripts/index.html'),# noqa - ('/site/support/bio-formats', 'https://docs.openmicroscopy.org/latest/bio-formats/'),# noqa - ('/site/support/bio-formats5', 'https://docs.openmicroscopy.org/latest/bio-formats5/'),# noqa - ('/site/support/bio-formats5.3', 'https://docs.openmicroscopy.org/latest/bio-formats5.3/'),# noqa - ('/site/support/bio-formats5.4', 'https://docs.openmicroscopy.org/latest/bio-formats5.4/'),# noqa - ('/site/support/bio-formats5.5', 'https://docs.openmicroscopy.org/latest/bio-formats5.5/'),# noqa - ('/site/support/omero', 'https://docs.openmicroscopy.org/latest/omero/'),# noqa - ('/site/support/omero5', 'https://docs.openmicroscopy.org/latest/omero5/'),# noqa - ('/site/support/omero5.0', 'https://docs.openmicroscopy.org/latest/omero5.0/'),# noqa - ('/site/support/omero5.1', 'https://docs.openmicroscopy.org/latest/omero5.1/'),# noqa - ('/site/support/omero5.2', 'https://docs.openmicroscopy.org/latest/omero5.2/'),# noqa - ('/site/support/omero5.3', 'https://docs.openmicroscopy.org/latest/omero5.3/'),# noqa - ('/site/support/ome-model', 'https://docs.openmicroscopy.org/latest/ome-model/'),# noqa - ('/site/support/file-formats', 'https://docs.openmicroscopy.org/latest/ome-model/'),# noqa - ('/site/support/file-formats/schemas/specifications/compliant-file-specification', 'https://docs.openmicroscopy.org/latest/ome-model/specifications/'),# noqa - ('/site/support/ome-tiff', 'https://docs.openmicroscopy.org/latest/ome-model/ome-tiff/'),# noqa - ('/site/support/ome-files-cpp', 'https://docs.openmicroscopy.org/latest/ome-files-cpp/'),# noqa - ('/site/support/contributing', 'https://docs.openmicroscopy.org/contributing/'),# noqa - ('/site/support/previous', 'https://docs.openmicroscopy.org'), - ('/info/OMERO.insight', 'https://docs.openmicroscopy.org/latest/omero/users/index.html'),# noqa - ('/info/OMERO.importer', 'https://docs.openmicroscopy.org/latest/omero/users/index.html'),# noqa - ('/info/OMERO.editor', 'https://docs.openmicroscopy.org/latest/omero/users/index.html'),# noqa - ('/info/OMERO.web', 'https://docs.openmicroscopy.org/latest/omero/users/index.html'),# noqa - ('/info/OMERO.server', 'https://docs.openmicroscopy.org/latest/omero/users/index.html'),# noqa - ('/info/permissions', 'https://docs.openmicroscopy.org/latest/omero/sysadmins/server-permissions.html'),# noqa - ('/info/demo', 'http://help.openmicroscopy.org/demo-server.html'), - ('/info/lists', 'http://lists.openmicroscopy.org.uk/mailman/listinfo/'), - ('/info/videos', 'https://www.youtube.com/channel/UCyySB9ZzNi8aBGYqcxSrauQ'),# noqa - ('/info/downgrade', 'https://docs.openmicroscopy.org/latest/omero/developers/Model/XsltTransformations.html'),# noqa - ('/info/flimfit', 'http://flimfit.org'),# noqa - ('/info/scripts', 'https://docs.openmicroscopy.org/latest/omero/developers/scripts/index.html'),# noqa - ('/info/bio-formats', 'https://docs.openmicroscopy.org/latest/bio-formats/'),# noqa - ('/info/slidebook', 'https://www.intelligent-imaging.com/technical-answers'),# noqa -] -content_uris = [ - ('/community', 'This page was generated by phpBB'), - ('/community/ucp.php?mode=login', ' Login'), - ('/community/viewtopic.php?f=6&t=8319', - 'UserId issues from Matlab.'), - ('/community/viewtopic.php?f=11&t=8320', - 'View topic - Release of Bio-Formats 5.5.3'), - ('/community/viewtopic.php?p=18303#p18303', - '
'), - ('/community/index.php', 'Index page'), - - ('/Schemas', 'Open Microscopy Environment Schemas'), - ('/Schemas/ROI', 'Open Microscopy Environment ROI Schemas'), - ('/Schemas/broken-link', 'Open Microscopy Environment Schemas'), - - ('/qa2', 'OMERO.qa provides support services'), - ('/qa2/qa/feedback/17777', 'Go back'), - ('/qa2/qa/upload', 'Uploading sample images'), - ('/qa2/qa/feedback/?status=1', - 'If you cannot view feedback you previously submitted'), - ('/qa2/registry/demo_account', 'Requesting a demo server account'), - ('/qa2/registry/statistic', 'File statistics.'), -] -content_uris_no_slash = [ - ('/Schemas/OME/2016-06/ome.xsd', 'Schema June 2016'), - ('/Schemas/OME/2015-01/ome.xsd', 'Schema January 2015'), - ('/Schemas/ROI/2015-01/ROI.xsd', 'Region of Interest'), - ('/XMLschemas/OME/FC/ome.xsd', 'The OME element is a container'), - ('/XMLschemas/CA/RC1/CA.xsd', - 'Conforms to w3c http://www.w3.org/2001/XMLSchema'), - ('/XMLschemas/STD/RC2/STD.xsd', 'Defines a semantic type'), -] - - -# Based on -# https://github.com/openmicroscopy/prod-playbooks/blob/master/www/playbook.yml -@pytest.mark.parametrize('host', hosts) -@pytest.mark.parametrize('uri,expect', redirect_uris) -@pytest.mark.parametrize("suffix", suffixes) -def test_redirect_with_slash(host, uri, expect, suffix): - r = requests.head('%s%s%s' % (host, uri, suffix)) - assert r.is_redirect - assert r.headers['Location'] == '%s%s' % (host, expect) - - -@pytest.mark.parametrize('host', hosts) -@pytest.mark.parametrize('uri,expect', external_uris) -@pytest.mark.parametrize("suffix", suffixes) -def test_redirect_external(host, uri, expect, suffix): - r = requests.head('%s%s%s' % (host, uri, suffix)) - assert r.is_redirect - assert r.headers['Location'] == expect - - -@pytest.mark.parametrize('host', hosts) -def test_404(host): - uri = '/non-existent/path' - r = requests.head('%s%s' % (host, uri)) - assert r.status_code == 404 - - -@pytest.mark.parametrize('host', hosts) -@pytest.mark.parametrize('uri,content', content_uris) -@pytest.mark.parametrize('suffix', suffixes) -def test_content(host, uri, content, suffix): - r = requests.get('%s%s%s' % (host, uri, suffix)) - assert content in r.text - - -@pytest.mark.parametrize('host', hosts) -@pytest.mark.parametrize('uri,content', content_uris_no_slash) -def test_content_no_slash(host, uri, content): - r = requests.get('%s%s' % (host, uri)) - assert content in r.text diff --git a/www/tests/requirements.txt b/www/tests/requirements.txt deleted file mode 100644 index 547de5c5..00000000 --- a/www/tests/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest -requests diff --git a/www/www-deploy.yml b/www/www-deploy.yml deleted file mode 100644 index 29c9f2c1..00000000 --- a/www/www-deploy.yml +++ /dev/null @@ -1,350 +0,0 @@ -# Install NGINX - -- name: Www deploy - hosts: www - - roles: - - role: ome.ssl_certificate - - role: ome.nginx_proxy - tags: nginxconf - - handlers: - - name: Reload nginx - listen: ssl certificate changed - become: true - ansible.builtin.service: - name: nginx - state: reloaded - - vars: - nginx_proxy_worker_processes: >- - {{ ((ansible_processor_count * ansible_processor_cores) - / 2) |round|int }} - nginx_proxy_worker_connections: 65000 - nginx_proxy_ssl: true - nginx_proxy_ssl_certificate: "{{ ssl_certificate_bundled_path }}" - nginx_proxy_ssl_certificate_key: "{{ ssl_certificate_key_path }}" - nginx_proxy_http2: true - nginx_proxy_force_ssl: false - nginx_proxy_404: "/404.html" - nginx_proxy_conf_http: - - "client_max_body_size 2g" - nginx_proxy_backends: - # Proxy for QA application - - location: /qa2 - server: https://www-legacy.openmicroscopy.org/qa2 - - location: /static - server: https://www-legacy.openmicroscopy.org - nginx_proxy_redirect_map_locations: - # TODO: change to 301 when we're happy - - location: "~ ^/(BIO-FORMATS)($|/)" - code: 302 - - location: "~ ^/(OME-FILES)($|/)" - code: 302 - - location: "~ ^/(OMERO)($|/)" - code: 302 - - location: "~ ^/(site)($|/)" - code: 302 - - location: "~ ^/(omero-blog)($|/)" - code: 302 - - location: "~ ^/(info)($|/)" - code: 302 - - location: "~ ^/(forums)($|/)" - code: 302 - - location: "~ ^/(XMLschemas)($|/)" - code: 302 - - location: "~ ^/(Schemas/Samples)($|/)" - code: 302 - nginx_proxy_redirect_map: - # by default redirect to the 404 page - - match: default - dest: /404.html - - match: "~/omero-blog.*" - dest: http://blog.openmicroscopy.org - - match: "~/site/?$" - dest: / - - match: "~/site/news/?$" - dest: /announcements - - # about - - match: "~/site/about/?$" - dest: /about - - match: "~/site/about/who-ome" - dest: /teams - - match: "~/site/about/licensing" - dest: /licensing - - match: "~/site/about/licensing-attribution(/.*)?$" - dest: /licensing - - match: "~/site/about/ome-contributors/?$" - dest: /contributors - - match: "~/site/about/partners/?$" - dest: /commercial-partners - - match: "~/site/about/development-teams/?$" - dest: /teams - - match: "~/site/about/development-teams/glencoe-software" - dest: https://www.glencoesoftware.com/team.html - - match: "~/site/about/publications" - dest: /citing-ome - - match: "~/site/about/what-omero/overview" - dest: /omero - - match: "~/site/about/(?.*)$" - dest: /about - - # products - - match: "~/site/products/?$" - dest: /products - - match: "~/site/products/omero/?$" - dest: /omero - - match: "~/site/products/omero/downloads/?$" - dest: /omero/downloads/ - - match: "~/site/products/omero/feature-list/?$" - dest: /omero/features/ - - match: "~/site/products/omero/big-images-support/?$" - dest: /omero/view/ - - match: "~/site/products/omero/secvuln/?$" - dest: /security/advisories/ - - match: "~/site/products/ome5/secvuln/?$" - dest: /security/advisories/ - - match: "~/site/products/omero/secvuln/(?.*[^/])/?$" - dest: /security/advisories/$link/ - - match: "~/site/products/bio-formats/?$" - dest: /bio-formats - - match: "~/site/products/bio-formats/downloads/?$" - dest: /bio-formats/downloads/ - - match: "~/site/products/ome-files-cpp/?$" - dest: /ome-files - - match: "~/site/products/(?.*)$" - dest: /products - - # community - - match: "~/forums/?$" - dest: https://forum.image.sc/c/data-management - - match: "~/site/community/?$" - dest: /support - - match: "~/site/community/mailing-lists/?$" - dest: /support - - match: "~/site/events(/.*)?$" - dest: /events - - match: "~/site/community/minutes/conference-calls/?$" - dest: /on-the-web - - match: "~/site/community/minutes/meetings\ - /12th-annual-users-meeting-2017" - dest: /events/12th-annual-users-meeting-2017.html - - match: "~/site/community/minutes/meetings\ - /11th-annual-users-meeting-2016" - dest: /events/11th-annual-users-meeting-2016.html - - match: "~/site/community/minutes/meetings/\ - 10th-annual-users-meeting-june-2015" - dest: /events/10th-annual-users-meeting-june-2015.html - - match: "~/site/community/minutes/meetings/\ - 9th-annual-users-meeting-june-2014" - dest: /events/9th-annual-users-meeting-june-2014.html - - match: "~/site/community/jobs/?$" - dest: /careers - - match: "~/site/community/scripts/?$" - dest: "https://docs.openmicroscopy.org/\ - latest/omero/developers/scripts/index.html" - - match: "~/site/community/(?.*)$" - dest: /support - - # support - - match: "~/site/support/?$" - dest: /docs - - - match: "~/site/support/bio-formats/?$" - dest: https://docs.openmicroscopy.org/latest/bio-formats/ - - match: "~/site/support/bio-formats/(?.*)$" - dest: https://docs.openmicroscopy.org/latest/bio-formats/$link - - match: "~/site/support/bio-formats5/?$" - dest: https://docs.openmicroscopy.org/latest/bio-formats5/ - - match: "~/site/support/bio-formats5/(?.*)$" - dest: https://docs.openmicroscopy.org/latest/bio-formats5/$link - - match: '~/site/support/bio-formats5\.3/?$' - dest: https://docs.openmicroscopy.org/latest/bio-formats5.3/ - - match: '~/site/support/bio-formats5\.3/(?.*)$' - dest: https://docs.openmicroscopy.org/latest/bio-formats5.3/$link - - match: '~/site/support/bio-formats5\.4/?$' - dest: https://docs.openmicroscopy.org/latest/bio-formats5.4/ - - match: '~/site/support/bio-formats5\.4/(?.*)$' - dest: https://docs.openmicroscopy.org/latest/bio-formats5.4/$link - - match: '~/site/support/bio-formats5\.5/?$' - dest: https://docs.openmicroscopy.org/latest/bio-formats5.5/ - - match: '~/site/support/bio-formats5\.5/(?.*)$' - dest: https://docs.openmicroscopy.org/latest/bio-formats5.5/$link - - - match: "~/site/support/omero/?$" - dest: https://docs.openmicroscopy.org/latest/omero/ - - match: "~/site/support/omero/(?.*)$" - dest: https://docs.openmicroscopy.org/latest/omero/$link - - match: "~/site/support/omero5/?$" - dest: https://docs.openmicroscopy.org/latest/omero5/ - - match: "~/site/support/omero5/(?.*)$" - dest: https://docs.openmicroscopy.org/latest/omero5/$link - - match: '~/site/support/omero5\.0/?$' - dest: https://docs.openmicroscopy.org/latest/omero5.0/ - - match: '~/site/support/omero5\.0/(?.*)$' - dest: https://docs.openmicroscopy.org/latest/omero5.0/$link - - match: '~/site/support/omero5\.1/?$' - dest: https://docs.openmicroscopy.org/latest/omero5.1/ - - match: '~/site/support/omero5\.1/(?.*)$' - dest: https://docs.openmicroscopy.org/latest/omero5.1/$link - - match: '~/site/support/omero5\.2/?$' - dest: https://docs.openmicroscopy.org/latest/omero5.2/ - - match: '~/site/support/omero5\.2/(?.*)$' - dest: https://docs.openmicroscopy.org/latest/omero5.2/$link - - match: '~/site/support/omero5\.3/?$' - dest: https://docs.openmicroscopy.org/latest/omero5.3/ - - match: '~/site/support/omero5\.3/(?.*)$' - dest: https://docs.openmicroscopy.org/latest/omero5.3/$link - - - match: "~/site/support/ome-model/?$" - dest: https://docs.openmicroscopy.org/latest/ome-model/ - - match: "~/site/support/ome-model/(?.*)$" - dest: https://docs.openmicroscopy.org/latest/ome-model/$link - - match: "~/site/support/file-formats/?$" - dest: https://docs.openmicroscopy.org/latest/ome-model/ - - match: "~/site/support/file-formats/schemas\ - /specifications/compliant-file-specification/?$" - dest: https://docs.openmicroscopy.org/latest/ome-model/specifications/ - - match: "~/site/support/ome-tiff/?$" - dest: https://docs.openmicroscopy.org/latest/ome-model/ome-tiff/ - - match: "~/site/support/ome-files-cpp/?$" - dest: https://docs.openmicroscopy.org/latest/ome-files-cpp/ - - match: "~/site/support/ome-files-cpp/(?.*)$" - dest: https://docs.openmicroscopy.org/latest/ome-files-cpp/$link - - match: "~/site/support/contributing/?$" - dest: https://docs.openmicroscopy.org/contributing/ - - match: "~/site/support/contributing/(?.*)$" - dest: https://docs.openmicroscopy.org/contributing/$link - - match: "~/site/support/previous(/.*)?$" - dest: https://docs.openmicroscopy.org - - match: "~/site/support/ome-artwork(/.*)?$" - dest: /artwork - - # uppercase alias - - match: "~/BIO-FORMATS$" - dest: /bio-formats - - match: "~/BIO-FORMATS/(?.*)$" - dest: /bio-formats/$link - - match: "~/OME-FILES$" - dest: /ome-files - - match: "~/OME-FILES/(?.*)$" - dest: /ome-files/$link - - match: "~/OMERO$" - dest: /omero - - match: "~/OMERO/(?.*)$" - dest: /omero/$link - - # info - - match: "~/info/vulnerabilities/?$" - dest: /security/advisories/ - - match: "~/info/vulnerabilities/(?.*[^/])/?$" - dest: /security/advisories/$link/ - - match: "~/info/omero/?$" - dest: /omero - - match: "~/info/OMERO.insight/?$" - dest: https://docs.openmicroscopy.org/latest/omero/users/index.html - - match: "~/info/OMERO.importer/?$" - dest: https://docs.openmicroscopy.org/latest/omero/users/index.html - - match: "~/info/OMERO.editor/?$" - dest: https://docs.openmicroscopy.org/latest/omero/users/index.html - - match: "~/info/OMERO.web/?$" - dest: https://docs.openmicroscopy.org/latest/omero/users/index.html - - match: "~/info/OMERO.server" - dest: https://docs.openmicroscopy.org/latest/omero/users/index.html - - match: "~/info/permissions" - dest: "https://docs.openmicroscopy.org/latest\ - /omero/sysadmins/server-permissions.html" - - match: "~/info/demo" - dest: http://help.openmicroscopy.org/demo-server.html - - match: "~/info/cls" - dest: /omero/downloads/ - - match: "~/info/download" - dest: /omero/downloads/ - - match: "~/info/downloads" - dest: /omero/downloads/ - - match: "~/info/lists" - dest: http://lists.openmicroscopy.org.uk/mailman/listinfo/ - - match: "~/info/videos" - dest: https://www.youtube.com/channel/UCyySB9ZzNi8aBGYqcxSrauQ - - match: "~/info/attribution" - dest: /licensing/ - - match: "~/info/downgrade" - dest: "https://docs.openmicroscopy.org/latest/\ - omero/developers/Model/XsltTransformations.html" - - match: "~/info/flimfit" - dest: http://flimfit.org - - match: "~/info/scripts" - dest: "https://docs.openmicroscopy.org/latest/\ - omero/developers/scripts/index.html" - - match: "~/info/bio-formats(/.*)?$" - dest: https://docs.openmicroscopy.org/latest/bio-formats/ - - match: "~/info/slidebook" - dest: https://www.intelligent-imaging.com/technical-answers - - match: "~/info/(.*)?$" - dest: /site-map - - # Legacy XMLschemas endpoint - - match: "~/XMLschemas/(?.*[^/])?$" - dest: /Schemas/$link - - # Legacy schemas samples - - match: - "~/Schemas/Samples/2013-06/bioformats-artificial/\ - multi-channel-4D-series.ome.tif.zip" - dest: - "https://downloads.openmicroscopy.org/images/OME-TIFF/\ - 2013-06/bioformats-artificial/multi-channel-4D-series.ome.tif" - - match: "~/Schemas/Samples/2015-01/set-1-meta-companion" - dest: - https://downloads.openmicroscopy.org/images/OME-TIFF/2015-01/companion/ - - match: - "~/Schemas/Samples/2015-01/bioformats-artificial/\ - multi-channel-time-series.ome.tif.zip" - dest: - "https://downloads.openmicroscopy.org/images/OME-TIFF/\ - 2015-01/bioformats-artificial/multi-channel-time-series.ome.tif" - - match: "~/Schemas/Samples/(?.*)?$" - dest: https://downloads.openmicroscopy.org/images/OME-TIFF/$link - - match: "~/Schemas/Samples" - dest: https://downloads.openmicroscopy.org/images/ - - nginx_proxy_direct_locations: - - location: "/" - root: "/var/www/www.openmicroscopy.org/html" - index: index.html - - - location: "^~ /Schemas/Documentation/Generated/" - alias: /var/www/schemas_documentation/ - - - location: "/Schemas/Transforms/" - root: "/var/www/www.openmicroscopy.org/html/" - custom: - - autoindex on - - # Static copy of old phpBB forums: treat query params as part of filename - - location: "~ ^/community/style.php.*" - root: /var/www/phpbbforum/www.openmicroscopy.org - custom: - - try_files $request_uri $uri =404 - - default_type text/css - - location: "~ ^/community/?$" - redirect301: /community/index.php - - location: /community - root: /var/www/phpbbforum/www.openmicroscopy.org - custom: - # Need to exclude extra query parameters in incoming external links - # e.g. sid= - # If an exact match isn't found try just these parameters: - # [f, t, p], [f, t], [f] - - >- - try_files - $request_uri - $uri?f=$arg_f&t=$arg_t&p=$arg_p - $uri?f=$arg_f&t=$arg_t - $uri?f=$arg_f - =404 - - default_type text/html - -- name: Import www-static playbook - import_playbook: www-static.yml diff --git a/www/www-static.yml b/www/www-static.yml deleted file mode 100644 index f3c511c3..00000000 --- a/www/www-static.yml +++ /dev/null @@ -1,82 +0,0 @@ -# Update the static website - -- name: Www static - hosts: www - - pre_tasks: - - name: Check if phpbbforum already unzipped - ansible.builtin.stat: - path: "{{ phpbbforum_style_file }}" - register: _phpbbforum_style_file_st - - - name: Check if schemas_documentation already unzipped - ansible.builtin.stat: - path: "{{ schemas_doc_file }}" - register: _schemas_doc_file_st - - roles: - - role: ome.deploy_archive - become: true - deploy_archive_dest_dir: /var/www - deploy_archive_src_url: "https://downloads.openmicroscopy.org/web-\ - archive/phpbbforum-20190718.tar.gz" - deploy_archive_sha256: "e9d7a7eefbacf42ddbdf92b201584913cb6d94ec\ - 331750f811232b2e91aa5b40" - # This file is patched later so only unzip if it doesn't exist - when: not _phpbbforum_style_file_st.stat.exists - - - role: ome.deploy_archive - become: true - deploy_archive_dest_dir: /var/www - deploy_archive_src_url: "https://downloads.openmicroscopy.org/web-\ - archive/schemas_documentation-20211213.tar.gz" - deploy_archive_sha256: "27cc5def458112a2e259484906f2bc8c0e0e2bd\ - 0a728b0a478302537d67117ec" - # This file is patched later so only unzip if it doesn't exist - when: not _schemas_doc_file_st.stat.exists - - tasks: - - name: Install deployment script - become: true - template: - src: files/deploy - dest: /usr/local/bin/deploy - mode: 0555 - - - name: Install Cron daemon - become: true - ansible.builtin.yum: - name: cronie - state: installed - - - name: Add cron job updating the website - become: true - ansible.builtin.cron: - name: "Deploy the website" - special_time: hourly - job: >- - /usr/local/bin/deploy 2>&1 > - /dev/null || /usr/local/bin/deploy -f - - - name: Update static phpbb stylesheet - become: true - ansible.builtin.blockinfile: - block: | - form, - .quick-login, - .buttons, - #jumpbox~h3, - #jumpbox~p, - .headerspace~h3, - .headerspace~p, - ul.linklist.rightside, - ul.linklist li.rightside { - display: none; - } - marker: "/* {mark} ANSIBLE MANAGED BLOCK */" - path: "{{ phpbbforum_style_file }}" - - vars: - phpbbforum_style_file: "/var/www/phpbbforum/www.openmicroscopy\ - .org/community/style.php?id=7&lang=en" - schemas_doc_file: "/var/www/schemas_documentation/OME-2016-06/ome.html" From c1461feda8d6ad479f41863f09b44149139e8afd Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Mon, 18 Nov 2024 11:36:15 +0000 Subject: [PATCH 03/20] Adjust the demo email message for websockets --- omero/ome-demoserver.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/omero/ome-demoserver.yml b/omero/ome-demoserver.yml index a59bcc51..a176616c 100644 --- a/omero/ome-demoserver.yml +++ b/omero/ome-demoserver.yml @@ -260,6 +260,9 @@ 3. Once OMERO.insight is started, following the steps in the\n omero-guide [1], change the server address to\n demo.openmicroscopy.org\n and connect using the login details as above to import your data.\n + If insight errors with "...Please check the server address..."\n + change the server address to:\n + wss://demo.openmicroscopy.org/omero-ws\n 4. Use the walkthrough example [1] to get further ideas about how to start using OMERO.\n OME Team\n\n From 44efe7256e269caa6e192ef54c284218fe02efc3 Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Mon, 18 Nov 2024 14:13:22 +0000 Subject: [PATCH 04/20] Bump versions server,web,figure,tagsearch --- omero/ome-demoserver.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/omero/ome-demoserver.yml b/omero/ome-demoserver.yml index a176616c..846bbf44 100644 --- a/omero/ome-demoserver.yml +++ b/omero/ome-demoserver.yml @@ -210,9 +210,9 @@ vars: omero_figure_release: >- - {{ omero_figure_release_override | default('6.2.2') }} + {{ omero_figure_release_override | default('7.1.0') }} omero_figure_script_release: >- - {{ omero_figure_script_release_override | default('v6.2.2') }} + {{ omero_figure_script_release_override | default('v7.1.0') }} omero_fpbioimage_release: >- {{ omero_fpbioimage_release_override | default('0.4.1') }} omero_iviewer_release: >- @@ -222,14 +222,14 @@ omero_autotag_release: >- {{ omero_autotag_release_override | default('4.0.1') }} omero_tagsearch_release: >- - {{ omero_tagsearch_release_override | default('4.1.1') }} + {{ omero_tagsearch_release_override | default('4.2.0') }} omero_signup_release: >- {{ omero_signup_release_override | default('0.3.3') }} omero_server_release: >- - {{ omero_server_release_override | default('5.6.11') }} - omero_web_release: "{{ omero_web_release_override | default('5.26.0') }}" - omero_py_release: "{{ omero_py_release_override | default('5.19.2') }}" + {{ omero_server_release_override | default('5.6.13') }} + omero_web_release: "{{ omero_web_release_override | default('5.27.2') }}" + omero_py_release: "{{ omero_py_release_override | default('5.19.5') }}" # For https://github.com/openmicroscopy/ansible-role-java, # which is a dependency. java_jdk_install: true From 25533d4a3fc1fe6f4ab84a177e6875253addef50 Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Mon, 18 Nov 2024 14:27:40 +0000 Subject: [PATCH 05/20] Pin role versions cf. Seb's comment --- requirements.yml | 69 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/requirements.yml b/requirements.yml index 248a7e23..3d24397e 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,24 +1,47 @@ --- -- src: ome.omero_common -- src: ome.basedeps -- src: ome.java -- src: ome.python3_virtualenv -- src: ome.ice -- src: ome.postgresql -- src: ome.postgresql_client -- src: ome.deploy_archive -- src: ome.omero_server -- src: ome.omero_web -- src: ome.nginx -- src: ome.redis -- src: ome.selinux_utils -- src: ome.versioncontrol_utils -- src: ome.ssl_certificate -- src: nfs_mount -- src: nfs_share -- src: iptables_raw -- src: ome.cli_utils -- src: ome.docker -- src: ome.postgresql_backup -- src: ome.omero_user -- src: ome.lvm_partition +- name: ome.omero_common + version: 0.4.0 +- name: ome.basedeps + version: 1.3.2 +- name: ome.java + version: 2.2.0 +- name: ome.python3_virtualenv + version: 0.2.0 +- name: ome.ice + version: 4.4.4 +- name: ome.postgresql + version: 5.4.0 +- name: ome.postgresql_client + version: 0.4.3 +- name: ome.deploy_archive + version: 0.2.0 +- name: ome.omero_server + version: 6.1.0 +- name: ome.omero_web + version: 5.1.1 +- name: ome.nginx + version: 2.2.1 +- name: ome.redis + version: 1.3.0 +- name: ome.selinux_utils + version: 2.1.1 +- name: ome.versioncontrol_utils + version: 1.1.0 +- name: ome.ssl_certificate + version: 0.5.0 +- name: nfs_mount + version: 2.0.0 +- name: nfs_share + version: 1.0.0 +- name: iptables_raw + version: 0.4.0 +- name: ome.cli_utils + version: 1.2.5 +- name: ome.docker + version: 3.2.2 +- name: ome.postgresql_backup + version: 0.3.0 +- name: ome.omero_user + version: 0.4.0 +- name: ome.lvm_partition + version: 1.2.0 From efd4d1f6343d33d464d23d7c14a20a640e22f540 Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Mon, 18 Nov 2024 14:47:10 +0000 Subject: [PATCH 06/20] Rename omero folder to playbooks --- ansible.cfg | 2 +- molecule/ome-demoserver/molecule.yml | 2 +- .../files/confd-nested-proxy-exporters-web.conf | 0 {omero => playbooks}/files/confd-nested-proxy-exporters.conf | 0 {omero => playbooks}/files/learning-omero-web.conf | 0 {omero => playbooks}/files/omero-server-config-prometheus.omero | 0 {omero => playbooks}/files/sls-gallery-omero-web.conf | 0 {omero => playbooks}/ome-demoserver.yml | 0 {omero => playbooks}/roles/.gitignore | 0 {omero => playbooks}/templates/nginx-confd-websockets-conf.j2 | 0 .../templates/nginx-confdnestedincludes-minio-publicscratch.j2 | 0 .../nginx-confdnestedincludes-ns-pub-redirects-conf.j2 | 0 .../templates/nginx-confdnestedincludes-omero-ms-zarr.j2 | 0 .../templates/nginx-confdnestedincludes-omerows-conf.j2 | 0 .../templates/nginx-confdnestedincludes-ssl-conf.j2 | 0 {omero => playbooks}/templates/nginx-letsencrypt-conf.j2 | 0 {omero => playbooks}/templates/nginx-omero.conf.j2 | 0 {omero => playbooks}/templates/omero-web-config-for-cors.j2 | 0 {omero => playbooks}/templates/omero-web-config-for-webapps.j2 | 0 {omero => playbooks}/templates/omero-web-config-signup.j2 | 0 20 files changed, 2 insertions(+), 2 deletions(-) rename {omero => playbooks}/files/confd-nested-proxy-exporters-web.conf (100%) rename {omero => playbooks}/files/confd-nested-proxy-exporters.conf (100%) rename {omero => playbooks}/files/learning-omero-web.conf (100%) rename {omero => playbooks}/files/omero-server-config-prometheus.omero (100%) rename {omero => playbooks}/files/sls-gallery-omero-web.conf (100%) rename {omero => playbooks}/ome-demoserver.yml (100%) rename {omero => playbooks}/roles/.gitignore (100%) rename {omero => playbooks}/templates/nginx-confd-websockets-conf.j2 (100%) rename {omero => playbooks}/templates/nginx-confdnestedincludes-minio-publicscratch.j2 (100%) rename {omero => playbooks}/templates/nginx-confdnestedincludes-ns-pub-redirects-conf.j2 (100%) rename {omero => playbooks}/templates/nginx-confdnestedincludes-omero-ms-zarr.j2 (100%) rename {omero => playbooks}/templates/nginx-confdnestedincludes-omerows-conf.j2 (100%) rename {omero => playbooks}/templates/nginx-confdnestedincludes-ssl-conf.j2 (100%) rename {omero => playbooks}/templates/nginx-letsencrypt-conf.j2 (100%) rename {omero => playbooks}/templates/nginx-omero.conf.j2 (100%) rename {omero => playbooks}/templates/omero-web-config-for-cors.j2 (100%) rename {omero => playbooks}/templates/omero-web-config-for-webapps.j2 (100%) rename {omero => playbooks}/templates/omero-web-config-signup.j2 (100%) diff --git a/ansible.cfg b/ansible.cfg index 67e073b8..76048139 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -10,7 +10,7 @@ [defaults] # Galaxy roles -roles_path = ./omero/roles +roles_path = ./playbooks/roles # These tend to be annoying. retry_files_enabled = False diff --git a/molecule/ome-demoserver/molecule.yml b/molecule/ome-demoserver/molecule.yml index b09507f8..4ad4b21c 100644 --- a/molecule/ome-demoserver/molecule.yml +++ b/molecule/ome-demoserver/molecule.yml @@ -26,7 +26,7 @@ provisioner: name: ansible playbooks: prepare: ../prepare.yml - converge: ../../omero/ome-demoserver.yml + converge: ../../playbooks/ome-demoserver.yml options: v: true diff: true diff --git a/omero/files/confd-nested-proxy-exporters-web.conf b/playbooks/files/confd-nested-proxy-exporters-web.conf similarity index 100% rename from omero/files/confd-nested-proxy-exporters-web.conf rename to playbooks/files/confd-nested-proxy-exporters-web.conf diff --git a/omero/files/confd-nested-proxy-exporters.conf b/playbooks/files/confd-nested-proxy-exporters.conf similarity index 100% rename from omero/files/confd-nested-proxy-exporters.conf rename to playbooks/files/confd-nested-proxy-exporters.conf diff --git a/omero/files/learning-omero-web.conf b/playbooks/files/learning-omero-web.conf similarity index 100% rename from omero/files/learning-omero-web.conf rename to playbooks/files/learning-omero-web.conf diff --git a/omero/files/omero-server-config-prometheus.omero b/playbooks/files/omero-server-config-prometheus.omero similarity index 100% rename from omero/files/omero-server-config-prometheus.omero rename to playbooks/files/omero-server-config-prometheus.omero diff --git a/omero/files/sls-gallery-omero-web.conf b/playbooks/files/sls-gallery-omero-web.conf similarity index 100% rename from omero/files/sls-gallery-omero-web.conf rename to playbooks/files/sls-gallery-omero-web.conf diff --git a/omero/ome-demoserver.yml b/playbooks/ome-demoserver.yml similarity index 100% rename from omero/ome-demoserver.yml rename to playbooks/ome-demoserver.yml diff --git a/omero/roles/.gitignore b/playbooks/roles/.gitignore similarity index 100% rename from omero/roles/.gitignore rename to playbooks/roles/.gitignore diff --git a/omero/templates/nginx-confd-websockets-conf.j2 b/playbooks/templates/nginx-confd-websockets-conf.j2 similarity index 100% rename from omero/templates/nginx-confd-websockets-conf.j2 rename to playbooks/templates/nginx-confd-websockets-conf.j2 diff --git a/omero/templates/nginx-confdnestedincludes-minio-publicscratch.j2 b/playbooks/templates/nginx-confdnestedincludes-minio-publicscratch.j2 similarity index 100% rename from omero/templates/nginx-confdnestedincludes-minio-publicscratch.j2 rename to playbooks/templates/nginx-confdnestedincludes-minio-publicscratch.j2 diff --git a/omero/templates/nginx-confdnestedincludes-ns-pub-redirects-conf.j2 b/playbooks/templates/nginx-confdnestedincludes-ns-pub-redirects-conf.j2 similarity index 100% rename from omero/templates/nginx-confdnestedincludes-ns-pub-redirects-conf.j2 rename to playbooks/templates/nginx-confdnestedincludes-ns-pub-redirects-conf.j2 diff --git a/omero/templates/nginx-confdnestedincludes-omero-ms-zarr.j2 b/playbooks/templates/nginx-confdnestedincludes-omero-ms-zarr.j2 similarity index 100% rename from omero/templates/nginx-confdnestedincludes-omero-ms-zarr.j2 rename to playbooks/templates/nginx-confdnestedincludes-omero-ms-zarr.j2 diff --git a/omero/templates/nginx-confdnestedincludes-omerows-conf.j2 b/playbooks/templates/nginx-confdnestedincludes-omerows-conf.j2 similarity index 100% rename from omero/templates/nginx-confdnestedincludes-omerows-conf.j2 rename to playbooks/templates/nginx-confdnestedincludes-omerows-conf.j2 diff --git a/omero/templates/nginx-confdnestedincludes-ssl-conf.j2 b/playbooks/templates/nginx-confdnestedincludes-ssl-conf.j2 similarity index 100% rename from omero/templates/nginx-confdnestedincludes-ssl-conf.j2 rename to playbooks/templates/nginx-confdnestedincludes-ssl-conf.j2 diff --git a/omero/templates/nginx-letsencrypt-conf.j2 b/playbooks/templates/nginx-letsencrypt-conf.j2 similarity index 100% rename from omero/templates/nginx-letsencrypt-conf.j2 rename to playbooks/templates/nginx-letsencrypt-conf.j2 diff --git a/omero/templates/nginx-omero.conf.j2 b/playbooks/templates/nginx-omero.conf.j2 similarity index 100% rename from omero/templates/nginx-omero.conf.j2 rename to playbooks/templates/nginx-omero.conf.j2 diff --git a/omero/templates/omero-web-config-for-cors.j2 b/playbooks/templates/omero-web-config-for-cors.j2 similarity index 100% rename from omero/templates/omero-web-config-for-cors.j2 rename to playbooks/templates/omero-web-config-for-cors.j2 diff --git a/omero/templates/omero-web-config-for-webapps.j2 b/playbooks/templates/omero-web-config-for-webapps.j2 similarity index 100% rename from omero/templates/omero-web-config-for-webapps.j2 rename to playbooks/templates/omero-web-config-for-webapps.j2 diff --git a/omero/templates/omero-web-config-signup.j2 b/playbooks/templates/omero-web-config-signup.j2 similarity index 100% rename from omero/templates/omero-web-config-signup.j2 rename to playbooks/templates/omero-web-config-signup.j2 From 98f403654d48a1e52ed15c865e31df2bc1509860 Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Wed, 20 Nov 2024 17:31:28 +0000 Subject: [PATCH 07/20] Add tests --- molecule/tests/test_default.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 molecule/tests/test_default.py diff --git a/molecule/tests/test_default.py b/molecule/tests/test_default.py new file mode 100644 index 00000000..b534293c --- /dev/null +++ b/molecule/tests/test_default.py @@ -0,0 +1,33 @@ +import os +import pytest +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + +OMERO = '/opt/omero/server/OMERO.server/bin/omero' +OMERO_LOGIN = '-C -s localhost -u root -w omero' + + +@pytest.mark.parametrize("name", [ + 'nginx', + 'omero-server', + 'omero-web', + 'postgresql-16', +]) +def test_service_running_and_enabled(host, name): + service = host.service(name) + assert service.is_running + assert service.is_enabled + + +def test_omero_login(host): + with host.sudo('omero-server'): + host.check_output( + '/opt/omero/server/OMERO.server/bin/omero ' + 'login -C -s localhost -u root -w omero') + + +def test_omero_nginx_ssl(host): + out = host.check_output('curl -fkI https://localhost/') + assert 'Location: /webclient/' in out From 07c606d66b48abbcb6144ddb08a907ad01a8eaab Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Wed, 20 Nov 2024 17:19:52 +0000 Subject: [PATCH 08/20] Reformat requirements.yml --- requirements.yml | 69 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/requirements.yml b/requirements.yml index 3d24397e..b96e8576 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,47 +1,70 @@ --- -- name: ome.omero_common + +- src: ome.omero_common version: 0.4.0 -- name: ome.basedeps + +- src: ome.basedeps version: 1.3.2 -- name: ome.java + +- src: ome.java version: 2.2.0 -- name: ome.python3_virtualenv + +- src: ome.python3_virtualenv version: 0.2.0 -- name: ome.ice + +- src: ome.ice version: 4.4.4 -- name: ome.postgresql + +- src: ome.postgresql version: 5.4.0 -- name: ome.postgresql_client + +- src: ome.postgresql_client version: 0.4.3 -- name: ome.deploy_archive + +- src: ome.deploy_archive version: 0.2.0 -- name: ome.omero_server + +- src: ome.omero_server version: 6.1.0 -- name: ome.omero_web + +- src: ome.omero_web version: 5.1.1 -- name: ome.nginx + +- src: ome.nginx version: 2.2.1 -- name: ome.redis + +- src: ome.redis version: 1.3.0 -- name: ome.selinux_utils + +- src: ome.selinux_utils version: 2.1.1 -- name: ome.versioncontrol_utils + +- src: ome.versioncontrol_utils version: 1.1.0 -- name: ome.ssl_certificate + +- src: ome.ssl_certificate version: 0.5.0 -- name: nfs_mount + +- src: nfs_mount version: 2.0.0 -- name: nfs_share + +- src: nfs_share version: 1.0.0 -- name: iptables_raw + +- src: iptables_raw version: 0.4.0 -- name: ome.cli_utils + +- src: ome.cli_utils version: 1.2.5 -- name: ome.docker + +- src: ome.docker version: 3.2.2 -- name: ome.postgresql_backup + +- src: ome.postgresql_backup version: 0.3.0 -- name: ome.omero_user + +- src: ome.omero_user version: 0.4.0 -- name: ome.lvm_partition + +- src: ome.lvm_partition version: 1.2.0 From 2c28f48ce592d378447c74863d367823d5bbd55a Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Tue, 19 Nov 2024 17:08:20 +0000 Subject: [PATCH 09/20] Add tests --- molecule/ome-demoserver/molecule.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/molecule/ome-demoserver/molecule.yml b/molecule/ome-demoserver/molecule.yml index 4ad4b21c..48da6536 100644 --- a/molecule/ome-demoserver/molecule.yml +++ b/molecule/ome-demoserver/molecule.yml @@ -61,3 +61,4 @@ scenario: - destroy verifier: name: testinfra + directory: ../tests/ From ef36d3c37420e7b8165069ad1db1a9e2ff996390 Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Wed, 20 Nov 2024 18:59:20 +0000 Subject: [PATCH 10/20] Implement changes of ssl protocol and ciphers --- playbooks/templates/nginx-confdnestedincludes-ssl-conf.j2 | 4 ++-- playbooks/templates/nginx-omero.conf.j2 | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/playbooks/templates/nginx-confdnestedincludes-ssl-conf.j2 b/playbooks/templates/nginx-confdnestedincludes-ssl-conf.j2 index 2edd2564..cc323a39 100644 --- a/playbooks/templates/nginx-confdnestedincludes-ssl-conf.j2 +++ b/playbooks/templates/nginx-confdnestedincludes-ssl-conf.j2 @@ -5,8 +5,8 @@ ssl_certificate {{ ssl_certificate_bundled_path }}; ssl_certificate_key {{ ssl_certificate_key_path }}; # use default ssl_protocols and ssl_ciphers: -# ssl_protocols TLSv1 TLSv1.1 TLSv1.2; -# ssl_ciphers HIGH:!aNULL:!MD5; +# ssl_protocols TLSv1.2 TLSv1.3; # don't use SSLv3 ref: POODLE +# ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; # http://nginx.org/en/docs/http/configuring_https_servers.html ssl_prefer_server_ciphers on; diff --git a/playbooks/templates/nginx-omero.conf.j2 b/playbooks/templates/nginx-omero.conf.j2 index de7d9526..a00cac89 100644 --- a/playbooks/templates/nginx-omero.conf.j2 +++ b/playbooks/templates/nginx-omero.conf.j2 @@ -9,7 +9,8 @@ server { ssl_certificate {{ ssl_certificate_bundled_path }}; ssl_certificate_key {{ ssl_certificate_key_path }}; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_protocols TLSv1.2 TLSv1.3; # don't use SSLv3 ref: POODLE + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; if ($ssl_protocol = "") { rewrite ^/(.*) https://$host/$1 permanent; From fee1fdea5cbc1687f0ad82ef8205ba96d1d223ad Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Tue, 26 Nov 2024 12:39:02 +0000 Subject: [PATCH 11/20] Bump to v5 cf. J-M's comment --- .github/workflows/molecule.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index 8642c0f2..515a7b32 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -28,7 +28,7 @@ jobs: scenario: ${{fromJson(needs.list-scenarios.outputs.matrix)}} steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.9' - name: Install Ansible & Molecule From 05580ce6439dd1e162c838b31a6721d5cc212063 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 26 Nov 2024 14:51:29 +0000 Subject: [PATCH 12/20] Bump iviewer 0.15.0 --- playbooks/ome-demoserver.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/ome-demoserver.yml b/playbooks/ome-demoserver.yml index 846bbf44..e51005e5 100644 --- a/playbooks/ome-demoserver.yml +++ b/playbooks/ome-demoserver.yml @@ -216,7 +216,7 @@ omero_fpbioimage_release: >- {{ omero_fpbioimage_release_override | default('0.4.1') }} omero_iviewer_release: >- - {{ omero_iviewer_release_override | default('0.14.0') }} + {{ omero_iviewer_release_override | default('0.15.0') }} omero_parade_release: >- {{ omero_parade_release_override | default('0.2.4') }} omero_autotag_release: >- From 1bc8e6e53aa27f689b099f02691bd5391cc45464 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 27 Nov 2024 11:32:03 +0000 Subject: [PATCH 13/20] Fix cd playbooks in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 903bab7c..33948d1d 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Details - Run the `ome-demoserver.yml` playbook: ``` -cd omero +cd playbooks ansible-playbook --ask-become --become -i $PATH/TO/INVENTORY ome-demoserver.yml -l $YOUR-HOST-ADDRESS-OR-IP --diff ``` From cab5ad8c74122ce5db501de68cdd7b54b388e74e Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 16 Dec 2024 13:09:56 +0000 Subject: [PATCH 14/20] Bump omero-figure 7.2.0 --- playbooks/ome-demoserver.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/ome-demoserver.yml b/playbooks/ome-demoserver.yml index e51005e5..2548c499 100644 --- a/playbooks/ome-demoserver.yml +++ b/playbooks/ome-demoserver.yml @@ -210,9 +210,9 @@ vars: omero_figure_release: >- - {{ omero_figure_release_override | default('7.1.0') }} + {{ omero_figure_release_override | default('7.2.0') }} omero_figure_script_release: >- - {{ omero_figure_script_release_override | default('v7.1.0') }} + {{ omero_figure_script_release_override | default('v7.2.0') }} omero_fpbioimage_release: >- {{ omero_fpbioimage_release_override | default('0.4.1') }} omero_iviewer_release: >- From 19c0dfb6132e6a63f74e9720f5e21c35d9fb566f Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Fri, 13 Dec 2024 15:31:42 +0000 Subject: [PATCH 15/20] Adjust protocols and ciphers as per workshop&outreach --- playbooks/templates/nginx-confdnestedincludes-ssl-conf.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/playbooks/templates/nginx-confdnestedincludes-ssl-conf.j2 b/playbooks/templates/nginx-confdnestedincludes-ssl-conf.j2 index cc323a39..940af0e2 100644 --- a/playbooks/templates/nginx-confdnestedincludes-ssl-conf.j2 +++ b/playbooks/templates/nginx-confdnestedincludes-ssl-conf.j2 @@ -10,6 +10,10 @@ ssl_certificate_key {{ ssl_certificate_key_path }}; # http://nginx.org/en/docs/http/configuring_https_servers.html ssl_prefer_server_ciphers on; +ssl_protocols TLSv1.2 TLSv1.3; +ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; + + # HTTP Strict Transport Security (HSTS) add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; From f806daf26d06ed5e97a74dc93ea1ee7d3d7b69fe Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Fri, 13 Dec 2024 15:36:30 +0000 Subject: [PATCH 16/20] Remove unused jinja files for sls-gallery and learning --- playbooks/files/learning-omero-web.conf | 52 ---------------------- playbooks/files/sls-gallery-omero-web.conf | 48 -------------------- 2 files changed, 100 deletions(-) delete mode 100644 playbooks/files/learning-omero-web.conf delete mode 100644 playbooks/files/sls-gallery-omero-web.conf diff --git a/playbooks/files/learning-omero-web.conf b/playbooks/files/learning-omero-web.conf deleted file mode 100644 index 9bbf914e..00000000 --- a/playbooks/files/learning-omero-web.conf +++ /dev/null @@ -1,52 +0,0 @@ -server { - listen 80; - server_name learning.openmicroscopy.org; - return 301 https://$server_name$request_uri; -} - -server { - listen 443 ssl; - server_name learning.openmicroscopy.org; - - ssl_certificate /etc/pki/tls/certs/star_openmicroscopy_org.crt+bundle; - ssl_certificate_key /etc/pki/tls/private/star_openmicroscopy_org.key; - ssl_protocols TLSv1.2; - - add_header Strict-Transport-Security "max-age=31536000" always; - - sendfile on; - client_max_body_size 0; - - location / { - rewrite ^/$ /dundee/ permanent; - } - - location /schools { - rewrite ^ /dundee/ permanent; - } - - location /dundee { - error_page 502 @maintenance; - # checks for static file, if not found proxy to app - try_files $uri @proxy_to_app; - } - - location /dundee/static { - alias /opt/omero/web/OMERO.web/var/static; - } - - location @maintenance { - root /opt/omero/server/OMERO.server/etc/templates/error; - try_files $uri /maintainance.html =502; - } - - location @proxy_to_app { - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_buffering off; - - proxy_pass http://127.0.0.1:4080; - } -} diff --git a/playbooks/files/sls-gallery-omero-web.conf b/playbooks/files/sls-gallery-omero-web.conf deleted file mode 100644 index e24ec719..00000000 --- a/playbooks/files/sls-gallery-omero-web.conf +++ /dev/null @@ -1,48 +0,0 @@ -server { - listen 80; - server_name sls-repo.openmicroscopy.org; - return 301 https://$server_name$request_uri; -} - -server { - listen 443 ssl; - server_name sls-repo.openmicroscopy.org; - - ssl_certificate /etc/pki/tls/certs/star_openmicroscopy_org.crt+bundle; - ssl_certificate_key /etc/pki/tls/private/star_openmicroscopy_org.key; - ssl_protocols TLSv1.2; - - add_header Strict-Transport-Security "max-age=31536000" always; - - sendfile on; - client_max_body_size 0; - - location / { - rewrite ^/$ /ome-sls/ permanent; - } - - location /ome-sls { - error_page 502 @maintenance; - # checks for static file, if not found proxy to app - try_files $uri @proxy_to_app; - } - - location /ome-sls/static { - alias /opt/omero/web/OMERO.web/var/static; - } - - location @maintenance { - root /opt/omero/server/OMERO.server/etc/templates/error; - try_files $uri /maintainance.html =502; - } - - location @proxy_to_app { - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_buffering off; - - proxy_pass http://127.0.0.1:4080; - } -} From 02723b461745dfa2b3510fc94bb40276e2867a5e Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Fri, 13 Dec 2024 15:39:15 +0000 Subject: [PATCH 17/20] Introduce nginx_version: 1.26.2 variable in demo playbook --- playbooks/ome-demoserver.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/ome-demoserver.yml b/playbooks/ome-demoserver.yml index 2548c499..386116fe 100644 --- a/playbooks/ome-demoserver.yml +++ b/playbooks/ome-demoserver.yml @@ -209,6 +209,7 @@ force: true vars: + nginx_version: 1.26.2 omero_figure_release: >- {{ omero_figure_release_override | default('7.2.0') }} omero_figure_script_release: >- From 9005c5f4316567eb42f97625c5129d1531a0e1d8 Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Mon, 16 Dec 2024 16:08:02 +0000 Subject: [PATCH 18/20] Bump web and server on demo --- playbooks/ome-demoserver.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/ome-demoserver.yml b/playbooks/ome-demoserver.yml index 386116fe..bcfc7d42 100644 --- a/playbooks/ome-demoserver.yml +++ b/playbooks/ome-demoserver.yml @@ -228,8 +228,8 @@ {{ omero_signup_release_override | default('0.3.3') }} omero_server_release: >- - {{ omero_server_release_override | default('5.6.13') }} - omero_web_release: "{{ omero_web_release_override | default('5.27.2') }}" + {{ omero_server_release_override | default('5.6.14') }} + omero_web_release: "{{ omero_web_release_override | default('5.28.0') }}" omero_py_release: "{{ omero_py_release_override | default('5.19.5') }}" # For https://github.com/openmicroscopy/ansible-role-java, # which is a dependency. From 57171f3d0680984c2e38b1a12b7fbba46750c9a9 Mon Sep 17 00:00:00 2001 From: pwalczysko Date: Mon, 16 Dec 2024 16:34:54 +0000 Subject: [PATCH 19/20] Change the target owner of the figure_scripts dir to be omero-server --- playbooks/ome-demoserver.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/playbooks/ome-demoserver.yml b/playbooks/ome-demoserver.yml index bcfc7d42..cdbc699a 100644 --- a/playbooks/ome-demoserver.yml +++ b/playbooks/ome-demoserver.yml @@ -193,7 +193,8 @@ state: directory mode: 0755 recurse: true - owner: root + owner: "omero-server" + group: "omero-server" - name: Download the Figure_To_Pdf.py script become: true From f61fe856fbf4f38819863de49301a0c1ca8d1424 Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 10 Jan 2025 11:54:43 +0000 Subject: [PATCH 20/20] Bump omero-figure 7.2.1 --- playbooks/ome-demoserver.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/ome-demoserver.yml b/playbooks/ome-demoserver.yml index cdbc699a..99db8aa4 100644 --- a/playbooks/ome-demoserver.yml +++ b/playbooks/ome-demoserver.yml @@ -212,9 +212,9 @@ vars: nginx_version: 1.26.2 omero_figure_release: >- - {{ omero_figure_release_override | default('7.2.0') }} + {{ omero_figure_release_override | default('7.2.1') }} omero_figure_script_release: >- - {{ omero_figure_script_release_override | default('v7.2.0') }} + {{ omero_figure_script_release_override | default('v7.2.1') }} omero_fpbioimage_release: >- {{ omero_fpbioimage_release_override | default('0.4.1') }} omero_iviewer_release: >-