Build ddev flat package for macOS (#15851)

Author: ofek

.github/workflows/build-ddev.yml

@@ -378,9 +378,6 @@ jobs:
     - name: Install PyOxidizer ${{ env.PYOXIDIZER_VERSION }}
       run: pip install pyoxidizer==${{ env.PYOXIDIZER_VERSION }}
 
-    - name: Install create-dmg
-      run: brew install create-dmg
-
     # TODO: Use the next official release after 0.22.0 by removing these 2 blocks, uncommenting
     # the following one, and changing the artifact name to reflect the next version. See:
     # https://github.com/indygreg/apple-platform-rs/issues/82
@@ -416,10 +413,14 @@ jobs:
       env:
         APPLE_DEVELOPER_ID_APPLICATION_CERTIFICATE: "${{ secrets.APPLE_DEVELOPER_ID_APPLICATION_CERTIFICATE }}"
         APPLE_DEVELOPER_ID_APPLICATION_PRIVATE_KEY: "${{ secrets.APPLE_DEVELOPER_ID_APPLICATION_PRIVATE_KEY }}"
+        APPLE_DEVELOPER_ID_INSTALLER_CERTIFICATE: "${{ secrets.APPLE_DEVELOPER_ID_INSTALLER_CERTIFICATE }}"
+        APPLE_DEVELOPER_ID_INSTALLER_PRIVATE_KEY: "${{ secrets.APPLE_DEVELOPER_ID_INSTALLER_PRIVATE_KEY }}"
         APPLE_APP_STORE_CONNECT_API_DATA: "${{ secrets.APPLE_APP_STORE_CONNECT_API_DATA }}"
       run: |-
-        echo "$APPLE_DEVELOPER_ID_APPLICATION_CERTIFICATE" > /tmp/certificate.pem
-        echo "$APPLE_DEVELOPER_ID_APPLICATION_PRIVATE_KEY" > /tmp/private-key.pem
+        echo "$APPLE_DEVELOPER_ID_APPLICATION_CERTIFICATE" > /tmp/certificate-application.pem
+        echo "$APPLE_DEVELOPER_ID_APPLICATION_PRIVATE_KEY" > /tmp/private-key-application.pem
+        echo "$APPLE_DEVELOPER_ID_INSTALLER_CERTIFICATE" > /tmp/certificate-installer.pem
+        echo "$APPLE_DEVELOPER_ID_INSTALLER_PRIVATE_KEY" > /tmp/private-key-installer.pem
         echo "$APPLE_APP_STORE_CONNECT_API_DATA" > /tmp/app-store-connect.json
 
     # We cannot use anchors because of https://github.com/actions/runner/issues/1182 and
@@ -444,8 +445,8 @@ jobs:
         script<<INNER
         for f in bin/*; do
           rcodesign sign -vv \
-          --pem-source /tmp/certificate.pem \
-          --pem-source /tmp/private-key.pem \
+          --pem-source /tmp/certificate-application.pem \
+          --pem-source /tmp/private-key-application.pem \
           --code-signature-flags runtime \
           "$f"
         done
@@ -535,65 +536,53 @@ jobs:
           fi
         done
 
-    - name: Build app bundle
+    - name: Build universal binary
       run: >-
-        pyoxidizer build macos_app_bundle
+        pyoxidizer build macos_universal_binary
         --release
         --var version ${{ env.VERSION }}
 
-    - name: Stage app bundle
-      id: stage
+    - name: Prepare universal binary
+      id: binary
       run: |-
-        mkdir staged
-        mkdir signed
-        mv build/*/release/*/*.app staged
-        app_bundle="$(ls staged)"
-        app_name="${app_bundle:0:${#app_bundle}-4}"
+        binary=$(echo build/*/release/*/${{ env.APP_NAME }})
+        chmod +x "$binary"
+        echo "path=$binary" >> "$GITHUB_OUTPUT"
 
-        echo "app-bundle=$app_bundle" >> "$GITHUB_OUTPUT"
-        echo "app-name=$app_name-${{ env.VERSION }}.dmg" >> "$GITHUB_OUTPUT"
-        echo "dmg-file=$app_name-${{ env.VERSION }}.dmg" >> "$GITHUB_OUTPUT"
-
-    - name: Sign app bundle
+    - name: Build PKG
       run: >-
-        rcodesign sign -vv
-        --pem-source /tmp/certificate.pem
-        --pem-source /tmp/private-key.pem
-        "staged/${{ steps.stage.outputs.app-bundle }}"
-        "signed/${{ steps.stage.outputs.app-bundle }}"
+        python release/macos/build_pkg.py
+        --binary ${{ steps.binary.outputs.path }}
+        --version ${{ env.VERSION }}
+        staged
 
-    - name: Create DMG
-      run: >-
-        create-dmg
-        --volname "${{ steps.stage.outputs.app-name }}"
-        --hide-extension "${{ steps.stage.outputs.app-bundle }}"
-        --window-pos 200 120
-        --window-size 800 400
-        --icon-size 100
-        --app-drop-link 600 185
-        "${{ steps.stage.outputs.dmg-file }}"
-        signed
-
-    - name: Sign DMG
+    - name: Stage PKG
+      id: pkg
+      run: |-
+        mkdir signed
+        pkg_file="$(ls staged)"
+        echo "path=$pkg_file" >> "$GITHUB_OUTPUT"
+
+    - name: Sign PKG
       run: >-
         rcodesign sign -vv
-        --pem-source /tmp/certificate.pem
-        --pem-source /tmp/private-key.pem
-        "${{ steps.stage.outputs.dmg-file }}"
-        "${{ steps.stage.outputs.dmg-file }}"
+        --pem-source /tmp/certificate-installer.pem
+        --pem-source /tmp/private-key-installer.pem
+        "staged/${{ steps.pkg.outputs.path }}"
+        "signed/${{ steps.pkg.outputs.path }}"
 
-    - name: Notarize DMG
+    - name: Notarize PKG
       run: >-
         rcodesign notary-submit
         --api-key-path /tmp/app-store-connect.json
         --staple
-        "${{ steps.stage.outputs.dmg-file }}"
+        "signed/${{ steps.pkg.outputs.path }}"
 
     - name: Upload installer
       uses: actions/upload-artifact@v3
       with:
         name: installers
-        path: ddev/${{ steps.stage.outputs.dmg-file }}
+        path: ddev/signed/${{ steps.pkg.outputs.path }}
 
   publish:
     name: Publish release

ddev/pyoxidizer.bzl

@@ -67,17 +67,7 @@ def make_exe_installer():
     return bundle
 
 
-def make_macos_app_bundle():
-    # https://gregoryszorc.com/docs/pyoxidizer/main/tugger_starlark_type_macos_application_bundle_builder.html
-    bundle = MacOsApplicationBundleBuilder(DISPLAY_NAME)
-    bundle.set_info_plist_required_keys(
-        display_name=DISPLAY_NAME,
-        identifier="com.datadoghq." + APP_NAME,
-        version=VERSION,
-        signature=APP_NAME,
-        executable=APP_NAME,
-    )
-
+def make_macos_universal_binary():
     # https://gregoryszorc.com/docs/pyoxidizer/main/tugger_starlark_type_apple_universal_binary.html
     universal = AppleUniversalBinary(APP_NAME)
 
@@ -86,12 +76,10 @@ def make_macos_app_bundle():
 
     m = FileManifest()
     m.add_file(universal.to_file_content())
-    bundle.add_macos_manifest(m)
-
-    return bundle
+    return m
 
 
 register_target("windows_installers", make_exe_installer, default=True)
-register_target("macos_app_bundle", make_macos_app_bundle)
+register_target("macos_universal_binary", make_macos_universal_binary)
 
 resolve_targets()

ddev/pyproject.toml

@@ -64,6 +64,9 @@ git_describe_command = ["git", "describe", "--dirty", "--tags", "--long", "--mat
 [tool.hatch.build.hooks.vcs]
 version-file = "src/ddev/_version.py"
 
+[tool.hatch.build.targets.sdist]
+include = ["src"]
+
 [tool.hatch.build.targets.app]
 scripts = ["ddev"]
 

ddev/release/README.md

@@ -0,0 +1,5 @@
+# ddev release assets
+
+-----
+
+This directory stores files related to building binaries and installers for each platform.

ddev/release/macos/build_pkg.py

@@ -0,0 +1,125 @@
+# (C) Datadog, Inc. 2023-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+"""
+This script must be run from within the `ddev` directory.
+
+At a high level, the goal is to have a directory that emulates the full path structure of the
+target machine which then gets packaged by tools that are only available on macOS.
+"""
+from __future__ import annotations
+
+import argparse
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+REPO_DIR = Path.cwd().parent
+ASSETS_DIR = Path(__file__).parent / 'pkg'
+IDENTIFIER = 'com.datadoghq.ddev'
+COMPONENT_PACKAGE_NAME = f'{IDENTIFIER}.pkg'
+README = """\
+<!DOCTYPE html>
+<html>
+<head></head>
+<body>
+  <p>This will install ddev v{version} globally.</p>
+
+  <p>For more information on installing and upgrading ddev, see our <a href="https://datadoghq.dev/integrations-core/setup/#ddev">Installation Guide</a>.</p>
+</body>
+</html>
+"""  # noqa: E501
+
+
+def run_command(command: list[str]) -> None:
+    process = subprocess.run(command)
+    if process.returncode:
+        sys.exit(process.returncode)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('directory')
+    parser.add_argument('--binary', required=True)
+    parser.add_argument('--version', required=True)
+    args = parser.parse_args()
+
+    directory = Path(args.directory).absolute()
+    staged_binary = Path(args.binary).absolute()
+    binary_name = staged_binary.stem
+    version = args.version
+
+    with TemporaryDirectory() as d:
+        temp_dir = Path(d)
+
+        # This is where we assemble files required for builds
+        resources_dir = temp_dir / 'resources'
+        shutil.copytree(str(ASSETS_DIR / 'resources'), str(resources_dir))
+
+        resources_dir.joinpath('README.html').write_text(README.format(version=version), encoding='utf-8')
+        shutil.copy2(REPO_DIR / 'LICENSE', resources_dir)
+
+        # This is what gets shipped to users starting at / (the root directory)
+        root_dir = temp_dir / 'root'
+        root_dir.mkdir()
+
+        # This is where we globally install ddev. We choose to not offer per-user installs because we can't
+        # find out where the location is and therefore cannot add to PATH usually. For more information, see:
+        # https://github.com/aws/aws-cli/commit/f3c3eb8262786142a1712b6da5a1515ad9dc66c5
+        relative_binary_dir = Path('usr', 'local', binary_name, 'bin')
+        binary_dir = root_dir / relative_binary_dir
+        binary_dir.mkdir(parents=True)
+        shutil.copy2(staged_binary, binary_dir)
+
+        # This is how we add the installation directory to PATH and is also what Go does,
+        # although there are some caveats: https://apple.stackexchange.com/q/126725
+        path_file = root_dir / 'etc' / 'paths.d' / binary_name
+        path_file.parent.mkdir(parents=True)
+        path_file.write_text(f'/{relative_binary_dir}\n', encoding='utf-8')
+
+        # This is where we build the intermediate components
+        components_dir = temp_dir / 'components'
+        components_dir.mkdir()
+
+        run_command(
+            [
+                'pkgbuild',
+                '--root',
+                str(root_dir),
+                '--identifier',
+                IDENTIFIER,
+                '--version',
+                version,
+                '--install-location',
+                '/',
+                str(components_dir / COMPONENT_PACKAGE_NAME),
+            ]
+        )
+
+        # This is where we build the final artifact
+        build_dir = temp_dir / 'build'
+        build_dir.mkdir()
+        product_archive = build_dir / f'{binary_name}-{version}.pkg'
+
+        run_command(
+            [
+                'productbuild',
+                '--distribution',
+                str(ASSETS_DIR / 'distribution.xml'),
+                '--resources',
+                str(resources_dir),
+                '--package-path',
+                str(components_dir),
+                str(product_archive),
+            ]
+        )
+
+        # Copy the final artifact to the target directory
+        directory.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(product_archive, directory)
+
+
+if __name__ == '__main__':
+    main()

ddev/release/macos/pkg/distribution.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+(C) Datadog, Inc. 2023-present
+All rights reserved
+Licensed under a 3-clause BSD style license (see LICENSE)
+-->
+
+<installer-gui-script minSpecVersion="1">
+  <!--
+  https://developer.apple.com/library/archive/documentation/DeveloperTools/Reference/DistributionDefinitionRef/Chapters/Distribution_XML_Ref.html
+  -->
+  <title>Datadog Agent integration developer tool</title>
+  <license file="LICENSE" mime-type="text/plain"/>
+  <readme file="README.html" mime-type="text/html"/>
+  <background mime-type="image/png" file="icon.png" alignment="left" scaling="proportional"/>
+  <background-darkAqua mime-type="image/png" file="icon.png" alignment="left" scaling="proportional"/>
+  <options hostArchitectures="arm64,x86_64" customize="never" require-scripts="false"/>
+  <domains enable_localSystem="true"/>
+
+  <choices-outline>
+    <line choice="com.datadoghq.ddev.choice"/>
+  </choices-outline>
+  <choice title="ddev (universal)" id="com.datadoghq.ddev.choice">
+    <pkg-ref id="com.datadoghq.ddev.pkg"/>
+  </choice>
+
+  <pkg-ref id="com.datadoghq.ddev.pkg">com.datadoghq.ddev.pkg</pkg-ref>
+</installer-gui-script>

ddev/release/macos/pkg/resources/icon.png

@@ -0,0 +1,32 @@
+import os
+import subprocess
+from functools import cache
+from pathlib import Path
+
+MARKER = '<docs-insert-ddev-version>'
+SEMVER_PARTS = 3
+
+# Ignore the current documentation environment so that the version
+# command can execute as usual in the default build environment
+os.environ.pop('HATCH_ENV_ACTIVE', None)
+
+
+@cache
+def get_latest_version():
+    """This returns the latest version of ddev."""
+    ddev_root = Path.cwd() / 'ddev'
+    output = subprocess.check_output(['hatch', 'version'], cwd=str(ddev_root)).decode('utf-8').strip()
+
+    version = output.replace('dev', '')
+    parts = list(map(int, version.split('.')))
+    major, minor, patch = parts[:SEMVER_PARTS]
+    if len(parts) > SEMVER_PARTS:
+        patch -= 1
+
+    return f'{major}.{minor}.{patch}'
+
+
+def on_page_read_source(page, config):
+    """This inserts the latest version of ddev."""
+    with open(page.file.abs_src_path, encoding='utf-8') as f:
+        return f.read().replace(MARKER, get_latest_version())