Vanagon packaging

.github/workflows/build_vanagon.yml

@@ -0,0 +1,103 @@
+---
+name: Build openvox-server with vanagon
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Tag, branch, or SHA to build from'
+        required: true
+      branch:
+        description: >-
+          The release branch being built. Selects the default platform list.
+          Use `main` for 9.x builds and `8.x` for 8.x builds.
+        type: choice
+        options:
+          - main
+          - '8.x'
+        default: 'main'
+      platform_list:
+        description: >-
+          A comma-separated list of vanagon platforms to build for (e.g.
+          el-9-x86_64,ubuntu-24.04-amd64). Do not include spaces. If not
+          provided, will use the default server platform list for the branch.
+        required: false
+        type: string
+      vanagon_branch:
+        description: 'The branch of the vanagon repository to use'
+        required: false
+        type: string
+        default: 'main'
+      upload_to_s3:
+        description: >-
+          Whether to upload the built packages to S3. Note that the uberjar
+          archives are always uploaded to S3, because the per-platform builds
+          fetch them from there.
+        required: false
+        type: boolean
+        default: true
+      use_dev:
+        description: 'Use the "dev" branch of the shared-actions repo'
+        required: false
+        type: boolean
+        default: false
+
+# The uberjar jobs build the openvox-server-uberjar project once per variant,
+# see packaging/lib/server_packaging.rb for the platforms. The build jobs then
+# build openvox-server for every platform, fetching the archives from S3.
+jobs:
+  uberjar:
+    if: ${{ github.event.inputs.use_dev == 'false' }}
+    uses: 'openvoxproject/shared-actions/.github/workflows/build_vanagon.yml@main'
+    with:
+      ref: ${{ inputs.ref }}
+      project_name: 'openvox-server-uberjar'
+      platform_list: 'el-9-x86_64,redhatfips-9-x86_64'
+      vanagon_branch: ${{ inputs.vanagon_branch }}
+      upload_to_s3: true
+      working_directory: 'packaging'
+      branch: ${{ inputs.branch }}
+    secrets: inherit
+  build:
+    needs: uberjar
+    if: ${{ github.event.inputs.use_dev == 'false' }}
+    uses: 'openvoxproject/shared-actions/.github/workflows/build_vanagon.yml@main'
+    with:
+      ref: ${{ inputs.ref }}
+      project_name: 'openvox-server'
+      platform_list: ${{ inputs.platform_list }}
+      platforms_key: 'server-vanagon'
+      vanagon_branch: ${{ inputs.vanagon_branch }}
+      upload_to_s3: ${{ inputs.upload_to_s3 }}
+      working_directory: 'packaging'
+      branch: ${{ inputs.branch }}
+    secrets: inherit
+  uberjar_dev:
+    if: ${{ github.event.inputs.use_dev == 'true' }}
+    uses: 'openvoxproject/shared-actions/.github/workflows/build_vanagon.yml@dev'
+    with:
+      ref: ${{ inputs.ref }}
+      project_name: 'openvox-server-uberjar'
+      platform_list: 'el-9-x86_64,redhatfips-9-x86_64'
+      vanagon_branch: ${{ inputs.vanagon_branch }}
+      upload_to_s3: true
+      working_directory: 'packaging'
+      branch: ${{ inputs.branch }}
+    secrets: inherit
+  build_dev:
+    needs: uberjar_dev
+    if: ${{ github.event.inputs.use_dev == 'true' }}
+    uses: 'openvoxproject/shared-actions/.github/workflows/build_vanagon.yml@dev'
+    with:
+      ref: ${{ inputs.ref }}
+      project_name: 'openvox-server'
+      platform_list: ${{ inputs.platform_list }}
+      platforms_key: 'server-vanagon'
+      vanagon_branch: ${{ inputs.vanagon_branch }}
+      upload_to_s3: ${{ inputs.upload_to_s3 }}
+      working_directory: 'packaging'
+      branch: ${{ inputs.branch }}
+    secrets: inherit

Dockerfile

@@ -21,6 +21,7 @@ RUN dnf install -y --enablerepo=crb \
     readline-devel \
     rpm-build \
     ruby \
+    ruby-devel \
     sqlite-devel \
     wget \
     zlib \

Gemfile

@@ -1,7 +1,7 @@
 source ENV['GEM_SOURCE'] || 'https://rubygems.org'
 
 def location_for(place, fake_version = nil)
-  if place.is_a?(String) && place =~ /^(git[:@][^#]*)#(.*)/
+  if place.is_a?(String) && place =~ /^((?:git[:@]|https:)[^#]*)#(.*)/
     [fake_version, { :git => $1, :branch => $2, :require => false }].compact
   elsif place.is_a?(String) && place =~ /^file:\/\/(.*)/
     ['>= 0', { :path => File.expand_path($1), :require => false }]
@@ -11,8 +11,6 @@ def location_for(place, fake_version = nil)
 end
 
 gem 'public_suffix', '>= 4.0.7', '< 8'
-# 1.0.0 is the first OpenVoxProject release
-gem 'packaging', '~> 1.0', github: 'OpenVoxProject/packaging'
 gem 'rake', :group => [:development, :test]
 
 group :test do
@@ -26,6 +24,10 @@ group :test do
   gem 'docker-api', '>=1.31.0', '< 3'
 end
 
+group :packaging do
+  gem 'vanagon', *location_for(ENV['VANAGON_LOCATION'] || 'https://github.com/openvoxproject/vanagon#main')
+end
+
 group :release, optional: true do
   # usually we pin to  ~> 2.1, but some of the EoL beaker 6 dependencies require ancient faraday versions
   # it's all a huge pain and the beaker setup needs to be reworked

packaging/.gitignore

@@ -0,0 +1,4 @@
+output/
+tarballs/
+ext/build_metadata.*
+vanagon_hosts.log

packaging/README.md

@@ -0,0 +1,65 @@
+# openvox-server packaging
+
+This directory holds the [vanagon](https://github.com/openvoxproject/vanagon)
+configuration that builds the openvox-server rpm and deb packages. The packages
+are noarch and are built once per supported platform so each one carries the
+right java dependency, dist tag, and maintainer scripts.
+
+Two vanagon projects make up the build:
+
+1. `openvox-server-uberjar` builds everything that needs the JVM toolchain: the
+   `puppet-server-release.jar` uberjar, the vendored gems, and on the FIPS
+   variant the BouncyCastle FIPS jars. It runs once per variant, on the
+   platforms specified in `UBERJAR_PLATFORMS` in `lib/server_packaging.rb`, and
+   its output is a rooted archive, `openvox-server-uberjar-<version>.<platform>.tar.gz`,
+   that CI uploads to the artifacts S3 bucket.
+2. `openvox-server` builds the package for one platform. The `uberjar-tarball`
+   component fetches the matching uberjar archive (from S3 in CI, or from
+   `packaging/output` for local builds) and the `openvox-server` component
+   installs all authored content from `packaging/resources/`: CLI wrappers and
+   configs from `resources/files/`, and the systemd unit, defaults file, and
+   tmpfiles config rendered from the templates next to it.
+
+The package build for `SOURCE_TARBALL_PLATFORM` in `lib/server_packaging.rb` 
+also writes `openvox-server-<version>.tar.gz` to `packaging/output/`. That is
+the tarball downstream packagers such as the FreeBSD port build from: the jar
+and the authored content in the layout of the ezbake source tarball, under a
+`puppetserver-<version>/` top level directory.
+
+Both run through `rake "vox:build[<project>,<platform>]"` at the repo root,
+which is what the shared CI workflow calls.
+
+## Local builds
+
+Requirements: docker and `bundle install` at the repo root. The JDK and
+leiningen are installed inside the build container.
+
+```sh
+bundle exec rake "vox:build[openvox-server,el-9-x86_64]"
+bundle exec rake "vox:build[openvox-server,ubuntu-24.04-amd64]"
+bundle exec rake "vox:build[openvox-server,redhatfips-9-x86_64]"
+```
+
+A package build first builds the uberjar archive it needs when that is missing
+from `packaging/output/`. To build the archive on its own:
+
+```sh
+bundle exec rake "vox:build[openvox-server-uberjar,el-9-x86_64]"
+```
+
+Set `SERVER_TARBALL_BASE` to a `file://` or `https://` directory URL to fetch
+the archive from somewhere else. Packages land in `packaging/output/`.
+
+The Clojure dependencies can be rebuilt from source before the uberjar build
+with `DEP_REBUILD` (a comma separated subset), `DEP_REBUILD_BRANCH`,
+`DEP_REBUILD_ORG` and `FULL_DEP_REBUILD_BRANCH`, as with the ezbake build.
+
+## Service styles
+
+Two systemd service styles are maintained (see `packaging/resources/systemd/`).
+The `direct` style matches the ezbake 4.x packages shipped for OpenVox 9 and is
+the default there. It runs java straight from the unit with `Type=notify`, or
+`Type=notify-reload` on platforms with systemd 253 or newer. The `wrapper`
+style matches the ezbake 2.x packages shipped for OpenVox 8 and is the default
+for 8.x versions. It runs the service through the start/stop/reload CLI
+wrappers in `resources/files/wrapper/`. Override with `SERVER_SERVICE_STYLE`.

packaging/configs/components/openvox-server.rb

@@ -0,0 +1,131 @@
+require_relative '../../lib/server_packaging'
+
+# Assembles all the package contents from the repo checkout. The compiled
+# bits (uberjar, vendored gems, FIPS BC jars) come from the uberjar-tarball
+# component.
+component 'openvox-server' do |pkg, settings, platform|
+  pkg.url 'file://resources/files'
+  pkg.version settings[:package_version]
+
+  wrapper_style = settings[:service_style] == :wrapper
+
+  service_template = wrapper_style ? 'puppetserver-wrapper.service.erb' : 'puppetserver-direct.service.erb'
+  pkg.add_source "file://resources/systemd/#{service_template}", erb: true
+  pkg.add_source 'file://resources/systemd/puppetserver.sysconfig.erb', erb: true
+  pkg.add_source 'file://resources/cli-defaults.sh.erb', erb: true
+  pkg.add_source 'file://resources/tmpfiles.d/puppetserver.conf' if wrapper_style
+
+  pkg.build do
+    ["#{platform.sed} -i 's/@@VERSION@@/#{settings[:package_version]}/' bin/puppetserver"]
+  end
+
+  app_dir = '/opt/puppetlabs/server/apps/puppetserver'
+
+  pkg.install_file 'bin/puppetserver', "#{app_dir}/bin/puppetserver", mode: '0755'
+  pkg.link "../apps/puppetserver/bin/puppetserver", '/opt/puppetlabs/server/bin/puppetserver'
+  pkg.link "../server/apps/puppetserver/bin/puppetserver", '/opt/puppetlabs/bin/puppetserver'
+
+  cli_apps = %w[ca foreground gem irb prune ruby].map { |app| ["cli/#{app}", app] }
+  # The wrapper style ships its own version of reload alongside start and stop
+  cli_apps += wrapper_style ? %w[start stop reload].map { |app| ["wrapper/#{app}", app] } : [['cli/reload', 'reload']]
+  cli_apps.each do |source, app|
+    pkg.install_file source, "#{app_dir}/cli/apps/#{app}", mode: '0755'
+  end
+  pkg.install_file '../cli-defaults.sh', "#{app_dir}/cli/cli-defaults.sh", mode: '0755'
+  pkg.install_file 'wrapper/helper-functions.sh', "#{app_dir}/helper-functions.sh", mode: '0755' if wrapper_style
+
+  pkg.install_file 'system-config/services.d/bootstrap.cfg', "#{app_dir}/config/services.d/bootstrap.cfg"
+
+  %w[auth ca global metrics puppetserver web-routes webserver].each do |conf|
+    pkg.install_configfile "config/conf.d/#{conf}.conf", "/etc/puppetlabs/puppetserver/conf.d/#{conf}.conf"
+  end
+  pkg.install_configfile 'config/services.d/ca.cfg', '/etc/puppetlabs/puppetserver/services.d/ca.cfg'
+  pkg.install_configfile 'config/logback.xml', '/etc/puppetlabs/puppetserver/logback.xml'
+  pkg.install_configfile 'config/request-logging.xml', '/etc/puppetlabs/puppetserver/request-logging.xml'
+
+  pkg.install_file 'java.security.fips', '/opt/puppetlabs/server/data/puppetserver/java.security.fips' if platform.is_fips?
+
+  # The rendered service unit, defaults file, and cli-defaults.sh land in the
+  # workdir root, one level above this component's source directory
+  service_file = File.basename(service_template, '.erb')
+  pkg.install_service "../#{service_file}", '../puppetserver.sysconfig', 'puppetserver'
+  pkg.install_file '../puppetserver.conf', '/usr/lib/tmpfiles.d/puppetserver.conf' if wrapper_style
+
+  # Ordering only, so the jar is unpacked before the source tarball below is assembled
+  pkg.build_requires 'uberjar-tarball'
+
+  # One platform's build also emits the tarball downstream packagers such as
+  # the FreeBSD port build from. It carries the jar and the authored content
+  # in the layout of the ezbake source tarball, under the top level directory
+  # the port expects, and lands in output/ next to the packages.
+  if platform.name == ServerPackaging::SOURCE_TARBALL_PLATFORM
+    tarball_root = ServerPackaging.source_tarball_root(settings[:package_version])
+    tarball_name = ServerPackaging.source_tarball_name(settings[:package_version])
+    pkg.install do
+      [
+        "rm -rf ../#{tarball_root} && mkdir -p ../#{tarball_root}/ext/cli_defaults ../output",
+        "cp #{app_dir}/puppet-server-release.jar ../#{tarball_root}/",
+        "cp -r bin cli config system-config ../#{tarball_root}/ext/",
+        "cp ../cli-defaults.sh ../#{tarball_root}/ext/cli_defaults/",
+        "#{platform.tar} --sort=name --owner=0 --group=0 --numeric-owner --mtime=@$$SOURCE_DATE_EPOCH --clamp-mtime " \
+        "-C .. -czf ../output/#{tarball_name} #{tarball_root}",
+      ]
+    end
+  end
+
+  # User and group creation, kept identical to what the ezbake packages did.
+  # The rpm variant prefers uid and gid 52 when they are free.
+  if platform.is_rpm?
+    pkg.add_preinstall_action ['install', 'upgrade'],
+      [<<~HERE
+        getent group puppet >/dev/null || groupadd --system --force --gid 52 puppet
+        if getent passwd puppet > /dev/null; then
+          usermod --gid puppet --home /opt/puppetlabs/server/data/puppetserver \
+          --comment "puppetserver daemon" puppet || :
+        else
+          useradd_options=('--system' '--gid' 'puppet' '--home' '/opt/puppetlabs/server/data/puppetserver' '--shell' "$(which nologin)" '--comment' 'puppetserver daemon')
+          if ! getent passwd 52 > /dev/null; then
+            useradd_options+=('--uid' '52')
+          fi
+          useradd "${useradd_options[@]}" puppet || :
+        fi
+      HERE
+      ]
+  else
+    pkg.add_preinstall_action ['install', 'upgrade'],
+      [<<~HERE
+        getent group puppet > /dev/null || \
+          groupadd -r puppet || :
+        if getent passwd puppet > /dev/null; then
+          usermod --gid puppet \
+            --home /opt/puppetlabs/server/data/puppetserver \
+            --comment "puppetserver daemon" puppet || :
+        else
+          useradd -r --gid puppet \
+            --home /opt/puppetlabs/server/data/puppetserver --shell $(which nologin) \
+            --comment "puppetserver daemon" puppet || :
+        fi
+      HERE
+      ]
+  end
+
+  # Fresh install configuration, carried over from ext/ezbake.conf. The agent
+  # owns the puppet.conf and ssl directories this touches.
+  postinstall_commands = [
+    'install --owner=puppet --group=puppet -d /opt/puppetlabs/server/data/puppetserver/jruby-gems',
+    '/opt/puppetlabs/puppet/bin/puppet config set --section server vardir  /opt/puppetlabs/server/data/puppetserver',
+    '/opt/puppetlabs/puppet/bin/puppet config set --section server logdir  /var/log/puppetlabs/puppetserver',
+    '/opt/puppetlabs/puppet/bin/puppet config set --section server rundir  /var/run/puppetlabs/puppetserver',
+    '/opt/puppetlabs/puppet/bin/puppet config set --section server pidfile /var/run/puppetlabs/puppetserver/puppetserver.pid',
+    '/opt/puppetlabs/puppet/bin/puppet config set --section server codedir /etc/puppetlabs/code',
+    'usermod --home /opt/puppetlabs/server/data/puppetserver puppet',
+    'install --directory --owner=puppet --group=puppet --mode=775 /opt/puppetlabs/server/data',
+    'install --directory /etc/puppetlabs/puppet/ssl',
+    'chown -R puppet:puppet /etc/puppetlabs/puppet/ssl',
+    'find /etc/puppetlabs/puppet/ssl -type d -print0 | xargs -0 chmod 770',
+  ]
+  if platform.is_fips?
+    postinstall_commands << 'chown puppet:puppet /opt/puppetlabs/server/data/puppetserver/java.security.fips'
+  end
+  pkg.add_postinstall_action ['install'], postinstall_commands
+end