diff --git a/.github/workflows/build-ddev.yml b/.github/workflows/build-ddev.yml index b4ee5fc806585..b2ec23642a958 100644 --- a/.github/workflows/build-ddev.yml +++ b/.github/workflows/build-ddev.yml @@ -378,9 +378,6 @@ jobs: - name: Install PyOxidizer ${{ env.PYOXIDIZER_VERSION }} run: pip install pyoxidizer==${{ env.PYOXIDIZER_VERSION }} - - name: Install create-dmg - run: brew install create-dmg - # TODO: Use the next official release after 0.22.0 by removing these 2 blocks, uncommenting # the following one, and changing the artifact name to reflect the next version. See: # https://github.com/indygreg/apple-platform-rs/issues/82 @@ -416,10 +413,14 @@ jobs: env: APPLE_DEVELOPER_ID_APPLICATION_CERTIFICATE: "${{ secrets.APPLE_DEVELOPER_ID_APPLICATION_CERTIFICATE }}" APPLE_DEVELOPER_ID_APPLICATION_PRIVATE_KEY: "${{ secrets.APPLE_DEVELOPER_ID_APPLICATION_PRIVATE_KEY }}" + APPLE_DEVELOPER_ID_INSTALLER_CERTIFICATE: "${{ secrets.APPLE_DEVELOPER_ID_INSTALLER_CERTIFICATE }}" + APPLE_DEVELOPER_ID_INSTALLER_PRIVATE_KEY: "${{ secrets.APPLE_DEVELOPER_ID_INSTALLER_PRIVATE_KEY }}" APPLE_APP_STORE_CONNECT_API_DATA: "${{ secrets.APPLE_APP_STORE_CONNECT_API_DATA }}" run: |- - echo "$APPLE_DEVELOPER_ID_APPLICATION_CERTIFICATE" > /tmp/certificate.pem - echo "$APPLE_DEVELOPER_ID_APPLICATION_PRIVATE_KEY" > /tmp/private-key.pem + echo "$APPLE_DEVELOPER_ID_APPLICATION_CERTIFICATE" > /tmp/certificate-application.pem + echo "$APPLE_DEVELOPER_ID_APPLICATION_PRIVATE_KEY" > /tmp/private-key-application.pem + echo "$APPLE_DEVELOPER_ID_INSTALLER_CERTIFICATE" > /tmp/certificate-installer.pem + echo "$APPLE_DEVELOPER_ID_INSTALLER_PRIVATE_KEY" > /tmp/private-key-installer.pem echo "$APPLE_APP_STORE_CONNECT_API_DATA" > /tmp/app-store-connect.json # We cannot use anchors because of https://github.com/actions/runner/issues/1182 and @@ -444,8 +445,8 @@ jobs: script<- - pyoxidizer build macos_app_bundle + pyoxidizer build macos_universal_binary --release --var version ${{ env.VERSION }} - - name: Stage app bundle - id: stage + - name: Prepare universal binary + id: binary run: |- - mkdir staged - mkdir signed - mv build/*/release/*/*.app staged - app_bundle="$(ls staged)" - app_name="${app_bundle:0:${#app_bundle}-4}" + binary=$(echo build/*/release/*/${{ env.APP_NAME }}) + chmod +x "$binary" + echo "path=$binary" >> "$GITHUB_OUTPUT" - echo "app-bundle=$app_bundle" >> "$GITHUB_OUTPUT" - echo "app-name=$app_name-${{ env.VERSION }}.dmg" >> "$GITHUB_OUTPUT" - echo "dmg-file=$app_name-${{ env.VERSION }}.dmg" >> "$GITHUB_OUTPUT" - - - name: Sign app bundle + - name: Build PKG run: >- - rcodesign sign -vv - --pem-source /tmp/certificate.pem - --pem-source /tmp/private-key.pem - "staged/${{ steps.stage.outputs.app-bundle }}" - "signed/${{ steps.stage.outputs.app-bundle }}" + python release/macos/build_pkg.py + --binary ${{ steps.binary.outputs.path }} + --version ${{ env.VERSION }} + staged - - name: Create DMG - run: >- - create-dmg - --volname "${{ steps.stage.outputs.app-name }}" - --hide-extension "${{ steps.stage.outputs.app-bundle }}" - --window-pos 200 120 - --window-size 800 400 - --icon-size 100 - --app-drop-link 600 185 - "${{ steps.stage.outputs.dmg-file }}" - signed - - - name: Sign DMG + - name: Stage PKG + id: pkg + run: |- + mkdir signed + pkg_file="$(ls staged)" + echo "path=$pkg_file" >> "$GITHUB_OUTPUT" + + - name: Sign PKG run: >- rcodesign sign -vv - --pem-source /tmp/certificate.pem - --pem-source /tmp/private-key.pem - "${{ steps.stage.outputs.dmg-file }}" - "${{ steps.stage.outputs.dmg-file }}" + --pem-source /tmp/certificate-installer.pem + --pem-source /tmp/private-key-installer.pem + "staged/${{ steps.pkg.outputs.path }}" + "signed/${{ steps.pkg.outputs.path }}" - - name: Notarize DMG + - name: Notarize PKG run: >- rcodesign notary-submit --api-key-path /tmp/app-store-connect.json --staple - "${{ steps.stage.outputs.dmg-file }}" + "signed/${{ steps.pkg.outputs.path }}" - name: Upload installer uses: actions/upload-artifact@v3 with: name: installers - path: ddev/${{ steps.stage.outputs.dmg-file }} + path: ddev/signed/${{ steps.pkg.outputs.path }} publish: name: Publish release diff --git a/ddev/pyoxidizer.bzl b/ddev/pyoxidizer.bzl index 0abf244ff862d..ffdae72b89afb 100644 --- a/ddev/pyoxidizer.bzl +++ b/ddev/pyoxidizer.bzl @@ -67,17 +67,7 @@ def make_exe_installer(): return bundle -def make_macos_app_bundle(): - # https://gregoryszorc.com/docs/pyoxidizer/main/tugger_starlark_type_macos_application_bundle_builder.html - bundle = MacOsApplicationBundleBuilder(DISPLAY_NAME) - bundle.set_info_plist_required_keys( - display_name=DISPLAY_NAME, - identifier="com.datadoghq." + APP_NAME, - version=VERSION, - signature=APP_NAME, - executable=APP_NAME, - ) - +def make_macos_universal_binary(): # https://gregoryszorc.com/docs/pyoxidizer/main/tugger_starlark_type_apple_universal_binary.html universal = AppleUniversalBinary(APP_NAME) @@ -86,12 +76,10 @@ def make_macos_app_bundle(): m = FileManifest() m.add_file(universal.to_file_content()) - bundle.add_macos_manifest(m) - - return bundle + return m register_target("windows_installers", make_exe_installer, default=True) -register_target("macos_app_bundle", make_macos_app_bundle) +register_target("macos_universal_binary", make_macos_universal_binary) resolve_targets() diff --git a/ddev/pyproject.toml b/ddev/pyproject.toml index 1991b13c57b8e..6669a100be2dd 100644 --- a/ddev/pyproject.toml +++ b/ddev/pyproject.toml @@ -64,6 +64,9 @@ git_describe_command = ["git", "describe", "--dirty", "--tags", "--long", "--mat [tool.hatch.build.hooks.vcs] version-file = "src/ddev/_version.py" +[tool.hatch.build.targets.sdist] +include = ["src"] + [tool.hatch.build.targets.app] scripts = ["ddev"] diff --git a/ddev/release/README.md b/ddev/release/README.md new file mode 100644 index 0000000000000..02e6ce7ea83ed --- /dev/null +++ b/ddev/release/README.md @@ -0,0 +1,5 @@ +# ddev release assets + +----- + +This directory stores files related to building binaries and installers for each platform. diff --git a/ddev/release/macos/build_pkg.py b/ddev/release/macos/build_pkg.py new file mode 100644 index 0000000000000..fb46c4dc586fb --- /dev/null +++ b/ddev/release/macos/build_pkg.py @@ -0,0 +1,125 @@ +# (C) Datadog, Inc. 2023-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +""" +This script must be run from within the `ddev` directory. + +At a high level, the goal is to have a directory that emulates the full path structure of the +target machine which then gets packaged by tools that are only available on macOS. +""" +from __future__ import annotations + +import argparse +import shutil +import subprocess +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +REPO_DIR = Path.cwd().parent +ASSETS_DIR = Path(__file__).parent / 'pkg' +IDENTIFIER = 'com.datadoghq.ddev' +COMPONENT_PACKAGE_NAME = f'{IDENTIFIER}.pkg' +README = """\ + + + + +

This will install ddev v{version} globally.

+ +

For more information on installing and upgrading ddev, see our Installation Guide.

+ + +""" # noqa: E501 + + +def run_command(command: list[str]) -> None: + process = subprocess.run(command) + if process.returncode: + sys.exit(process.returncode) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('directory') + parser.add_argument('--binary', required=True) + parser.add_argument('--version', required=True) + args = parser.parse_args() + + directory = Path(args.directory).absolute() + staged_binary = Path(args.binary).absolute() + binary_name = staged_binary.stem + version = args.version + + with TemporaryDirectory() as d: + temp_dir = Path(d) + + # This is where we assemble files required for builds + resources_dir = temp_dir / 'resources' + shutil.copytree(str(ASSETS_DIR / 'resources'), str(resources_dir)) + + resources_dir.joinpath('README.html').write_text(README.format(version=version), encoding='utf-8') + shutil.copy2(REPO_DIR / 'LICENSE', resources_dir) + + # This is what gets shipped to users starting at / (the root directory) + root_dir = temp_dir / 'root' + root_dir.mkdir() + + # This is where we globally install ddev. We choose to not offer per-user installs because we can't + # find out where the location is and therefore cannot add to PATH usually. For more information, see: + # https://github.com/aws/aws-cli/commit/f3c3eb8262786142a1712b6da5a1515ad9dc66c5 + relative_binary_dir = Path('usr', 'local', binary_name, 'bin') + binary_dir = root_dir / relative_binary_dir + binary_dir.mkdir(parents=True) + shutil.copy2(staged_binary, binary_dir) + + # This is how we add the installation directory to PATH and is also what Go does, + # although there are some caveats: https://apple.stackexchange.com/q/126725 + path_file = root_dir / 'etc' / 'paths.d' / binary_name + path_file.parent.mkdir(parents=True) + path_file.write_text(f'/{relative_binary_dir}\n', encoding='utf-8') + + # This is where we build the intermediate components + components_dir = temp_dir / 'components' + components_dir.mkdir() + + run_command( + [ + 'pkgbuild', + '--root', + str(root_dir), + '--identifier', + IDENTIFIER, + '--version', + version, + '--install-location', + '/', + str(components_dir / COMPONENT_PACKAGE_NAME), + ] + ) + + # This is where we build the final artifact + build_dir = temp_dir / 'build' + build_dir.mkdir() + product_archive = build_dir / f'{binary_name}-{version}.pkg' + + run_command( + [ + 'productbuild', + '--distribution', + str(ASSETS_DIR / 'distribution.xml'), + '--resources', + str(resources_dir), + '--package-path', + str(components_dir), + str(product_archive), + ] + ) + + # Copy the final artifact to the target directory + directory.mkdir(parents=True, exist_ok=True) + shutil.copy2(product_archive, directory) + + +if __name__ == '__main__': + main() diff --git a/ddev/release/macos/pkg/distribution.xml b/ddev/release/macos/pkg/distribution.xml new file mode 100644 index 0000000000000..5a83b3668f242 --- /dev/null +++ b/ddev/release/macos/pkg/distribution.xml @@ -0,0 +1,28 @@ + + + + + + Datadog Agent integration developer tool + + + + + + + + + + + + + + + com.datadoghq.ddev.pkg + diff --git a/ddev/release/macos/pkg/resources/icon.png b/ddev/release/macos/pkg/resources/icon.png new file mode 100644 index 0000000000000..f1243d9c3fa94 Binary files /dev/null and b/ddev/release/macos/pkg/resources/icon.png differ diff --git a/docs/developer/.hooks/ddev_version.py b/docs/developer/.hooks/ddev_version.py new file mode 100644 index 0000000000000..fff56b41aa628 --- /dev/null +++ b/docs/developer/.hooks/ddev_version.py @@ -0,0 +1,32 @@ +import os +import subprocess +from functools import cache +from pathlib import Path + +MARKER = '' +SEMVER_PARTS = 3 + +# Ignore the current documentation environment so that the version +# command can execute as usual in the default build environment +os.environ.pop('HATCH_ENV_ACTIVE', None) + + +@cache +def get_latest_version(): + """This returns the latest version of ddev.""" + ddev_root = Path.cwd() / 'ddev' + output = subprocess.check_output(['hatch', 'version'], cwd=str(ddev_root)).decode('utf-8').strip() + + version = output.replace('dev', '') + parts = list(map(int, version.split('.'))) + major, minor, patch = parts[:SEMVER_PARTS] + if len(parts) > SEMVER_PARTS: + patch -= 1 + + return f'{major}.{minor}.{patch}' + + +def on_page_read_source(page, config): + """This inserts the latest version of ddev.""" + with open(page.file.abs_src_path, encoding='utf-8') as f: + return f.read().replace(MARKER, get_latest_version()) diff --git a/docs/developer/setup.md b/docs/developer/setup.md index 8d699ed83033d..ac43b05dde76d 100644 --- a/docs/developer/setup.md +++ b/docs/developer/setup.md @@ -108,24 +108,92 @@ To install certain command line tools, you'll need [pipx](https://github.com/pyp ### Installation -You have 3 options to install the CLI. +You have 4 options to install the CLI. -!!! warning - For either option, if you are on macOS/Linux do not use `sudo`! Doing so will result in a broken installation. +#### Installers -#### GitHub releases +=== "macOS" + === "GUI installer" + 1. In your browser, download the `.pkg` file: [ddev-.pkg](https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev-.pkg) + 2. Run your downloaded file and follow the on-screen instructions. + 3. Restart your terminal. + 4. To verify that the shell can find and run the `ddev` command in your `PATH`, use the following command. + + ``` + $ ddev --version + + ``` + === "Command line installer" + 1. Download the file using the `curl` command. The `-o` option specifies the file name that the downloaded package is written to. In this example, the file is written to `ddev-.pkg` in the current directory. + + ``` + curl -o ddev-.pkg https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev-.pkg + ``` + 2. Run the standard macOS [`installer`](https://ss64.com/osx/installer.html) program, specifying the downloaded `.pkg` file as the source. Use the `-pkg` parameter to specify the name of the package to install, and the `-target /` parameter for the drive in which to install the package. The files are installed to `/usr/local/ddev`, and an entry is created at `/etc/paths.d/ddev` that instructs shells to add the `/usr/local/ddev` directory to. You must include sudo on the command to grant write permissions to those folders. + + ``` + sudo installer -pkg ./ddev-.pkg -target / + ``` + 3. Restart your terminal. + 4. To verify that the shell can find and run the `ddev` command in your `PATH`, use the following command. + + ``` + $ ddev --version + + ``` -Each [release](https://github.com/DataDog/integrations-core/releases?q=ddev-&expanded=true) provides the following: +=== "Windows" + === "GUI installer" + 1. In your browser, download one the `.msi` files: + - [ddev--x64.msi](https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev--x64.msi) + - [ddev--x86.msi](https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev--x86.msi) + 2. Run your downloaded file and follow the on-screen instructions. + 3. Restart your terminal. + 4. To verify that the shell can find and run the `ddev` command in your `PATH`, use the following command. + + ``` + $ ddev --version + + ``` + === "Command line installer" + 1. Download and run the installer using the standard Windows [`msiexec`](https://learn.microsoft.com/en-us/windows-server/administration/windows-commands/msiexec) program, specifying one of the `.msi` files as the source. Use the `/passive` and `/i` parameters to request an unattended, normal installation. + + === "x64" + ``` + msiexec /passive /i https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev--x64.msi + ``` + === "x86" + ``` + msiexec /passive /i https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev--x86.msi + ``` + 2. Restart your terminal. + 3. To verify that the shell can find and run the `ddev` command in your `PATH`, use the following command. + + ``` + $ ddev --version + + ``` + +#### Standalone binaries + +After downloading the archive corresponding to your platform and architecture, extract the binary to a directory that is on your PATH and rename to `ddev`. -- Standalone binaries for Linux, Windows, and macOS -- Windows AMD64 (64-bit) MSI installer -- Windows x86 (32-bit) MSI installer -- Windows universal (AMD64+x86) EXE installer -- macOS DMG installer +=== "macOS" + - [ddev--aarch64-apple-darwin.tar.gz](https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev--aarch64-apple-darwin.tar.gz) + - [ddev--x86_64-apple-darwin.tar.gz](https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev--x86_64-apple-darwin.tar.gz) -#### Stable +=== "Windows" + - [ddev--x86_64-pc-windows-msvc.zip](https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev--x86_64-pc-windows-msvc.zip) + - [ddev--i686-pc-windows-msvc.zip](https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev--i686-pc-windows-msvc.zip) -The latest released version may be installed from [PyPI][]. +=== "Linux" + - [ddev--aarch64-unknown-linux-gnu.tar.gz](https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev--aarch64-unknown-linux-gnu.tar.gz) + - [ddev--x86_64-unknown-linux-gnu.tar.gz](https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev--x86_64-unknown-linux-gnu.tar.gz) + - [ddev--x86_64-unknown-linux-musl.tar.gz](https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev--x86_64-unknown-linux-musl.tar.gz) + - [ddev--i686-unknown-linux-gnu.tar.gz](https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev--i686-unknown-linux-gnu.tar.gz) + - [ddev--powerpc64le-unknown-linux-gnu.tar.gz](https://github.com/DataDog/integrations-core/releases/download/ddev-v/ddev--powerpc64le-unknown-linux-gnu.tar.gz) + +#### PyPI === "macOS" Remove any executables shown in the output of `which -a ddev` and make sure that there is no active virtual environment, then run: @@ -139,6 +207,9 @@ The latest released version may be installed from [PyPI][]. pipx install ddev --python /usr/local/bin/python3.9 ``` + !!! warning + Do not use `sudo` as it may result in a broken installation! + === "Windows" Run: @@ -153,6 +224,15 @@ The latest released version may be installed from [PyPI][]. pipx install ddev ``` + !!! warning + Do not use `sudo` as it may result in a broken installation! + +Upgrade at any time by running: + +``` +pipx upgrade ddev +``` + #### Development This is if you cloned [integrations-core][] and want to always use the version based on the current branch. @@ -169,6 +249,9 @@ This is if you cloned [integrations-core][] and want to always use the version b pipx install -e /path/to/integrations-core/ddev --python /usr/local/opt/python@3.9/bin/python3.9 ``` + !!! warning + Do not use `sudo` as it may result in a broken installation! + === "Windows" Run: @@ -183,6 +266,15 @@ This is if you cloned [integrations-core][] and want to always use the version b pipx install -e /path/to/integrations-core/ddev ``` + !!! warning + Do not use `sudo` as it may result in a broken installation! + +Re-sync dependencies at any time by running: + +``` +pipx upgrade ddev +``` + !!! note Be aware that this method does not keep track of dependencies so you will need to re-run the command if/when the required dependencies are changed. @@ -192,14 +284,6 @@ This is if you cloned [integrations-core][] and want to always use the version b ``` pipx inject ddev -e "/path/to/datadog_checks_dev" ``` - -### Upgrade - -Upgrade (or re-sync dependencies for [development](#development) versions) at any time by running: - -``` -pipx upgrade ddev -``` ### Configuration diff --git a/mkdocs.yml b/mkdocs.yml index 28d064a5e5f99..67ea437a2d851 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -122,6 +122,9 @@ plugins: - pdf-export: enabled_if_env: ENABLE_PDF_SITE_EXPORT +hooks: + - docs/developer/.hooks/ddev_version.py + markdown_extensions: # Built-in - markdown.extensions.abbr: