Merge pull request #102 from bastelfreak/ezbakefoo

Update release & build tasks

Author: bastelfreak

.gitignore

@@ -1,3 +1,4 @@
+output/
 
 # Leiningen
 /target/

tasks/build.rake

@@ -12,11 +12,30 @@ require 'tmpdir'
 # To be fixed one of these days. Relevant stuff:
 #   https://github.com/puppetlabs/ezbake/blob/aeb7735a16d2eecd389a6bd9e5c0cfc7c62e61a5/resources/puppetlabs/lein-ezbake/template/global/tasks/build.rake
 #   https://github.com/puppetlabs/ezbake/blob/aeb7735a16d2eecd389a6bd9e5c0cfc7c62e61a5/resources/puppetlabs/lein-ezbake/template/global/ext/fpm.rb
-deb_platforms = ENV['DEB_PLATFORMS'] || 'ubuntu-20.04,ubuntu-22.04,ubuntu-24.04,debian-10,debian-11,debian-12'
-rpm_platforms = ENV['RPM_PLATFORMS'] || 'el-8,el-9,el-10,sles-15,amazon-2023'
+deb_platforms = ENV['DEB_PLATFORMS'] || 'ubuntu-20.04,ubuntu-22.04,ubuntu-24.04,ubuntu-25.04,debian-11,debian-12,debian-13'
+rpm_platforms = ENV['RPM_PLATFORMS'] || 'el-8,el-9,el-10,sles-15,sles-16,amazon-2,amazon-2023,fedora-42,fedora-43'
 @debs = deb_platforms.split(',').map{ |p| "base-#{p.split('-').join}-i386.cow" }.join(' ')
 @rpms = rpm_platforms.split(',').map{ |p| "pl-#{p}-x86_64" }.join(' ')
 
+# The deps must be built in this order due to dependencies between them.
+# There is a circular dependency between clj-http-client and trapperkeeper-webserver-jetty10,
+# but only for tests, so the build *should* work.
+DEP_BUILD_ORDER = [
+  'clj-parent',
+  'clj-kitchensink',
+  'clj-i18n',
+  'comidi',
+  'jvm-ssl-utils',
+  'trapperkeeper',
+  'trapperkeeper-filesystem-watcher',
+  'trapperkeeper-webserver-jetty10',
+  'trapperkeeper-authorization',
+  'trapperkeeper-metrics',
+  'trapperkeeper-status',
+  'stockpile',
+  'structured-logging',
+].freeze
+
 def image_exists
   !`docker images -q #{@image}`.strip.empty?
 end
@@ -34,7 +53,7 @@ def teardown
 end
 
 def start_container(ezbake_dir)
-  run_command("docker run -d --name #{@container} -v .:/code -v #{ezbake_dir}:/ezbake #{@image} /bin/sh -c 'tail -f /dev/null'", silent: false, print_command: true)
+  run_command("docker run -d --name #{@container} -v .:/code -v #{ezbake_dir}:/deps #{@image} /bin/sh -c 'tail -f /dev/null'", silent: false, print_command: true)
 end
 
 def run(cmd)
@@ -60,24 +79,60 @@ namespace :vox do
         run_command("docker build -t ezbake-builder .", silent: false, print_command: true)
       end
 
-      puts "Checking out ezbake"
-      tmp = Dir.mktmpdir("ezbake")
-      ezbake_dir = "#{tmp}/ezbake"
-      ezbake_repo = ENV.fetch('EZBAKE_REPO', 'https://github.com/openvoxproject/ezbake')
-      ezbake_branch = ENV['EZBAKE_BRANCH'] || 'main'
-      run_command("git clone -b #{ezbake_branch} #{ezbake_repo} #{ezbake_dir}", silent: false, print_command: true)
+      libs_to_build_manually = {}
+      if ENV['EZBAKE_BRANCH'] && !ENV['EZBAKE_BRANCH'].strip.empty?
+        libs_to_build_manually['ezbake'] = {
+          :repo => ENV.fetch('EZBAKE_REPO', 'https://github.com/openvoxproject/ezbake'),
+          :branch => ENV.fetch('EZBAKE_BRANCH', 'main'),
+        }
+      end
+
+      deps_to_build = []
+      dep_branch = nil
+
+      full_rebuild_branch = ENV['FULL_DEP_REBUILD_BRANCH']
+      subset_list = (ENV['DEP_REBUILD'] || '').split(',').map(&:strip).reject(&:empty?)
+      subset_branch = ENV.fetch('DEP_REBUILD_BRANCH', 'main').to_s
+      rebuild_org = ENV.fetch('DEP_REBUILD_ORG', 'openvoxproject').to_s
+
+      if full_rebuild_branch && !full_rebuild_branch.strip.empty?
+        dep_branch = full_rebuild_branch.strip
+        deps_to_build = DEP_BUILD_ORDER.dup
+      elsif !subset_list.empty?
+        dep_branch = subset_branch
+        unknown = subset_list.reject { |lib| DEP_BUILD_ORDER.include?(lib) }
+        puts "WARNING: Unknown deps in DEP_REBUILD (will be ignored): #{unknown.join(', ')}" unless unknown.empty?
+        deps_to_build = DEP_BUILD_ORDER.select { |lib| subset_list.include?(lib) }
+      end
+
+      deps_to_build.each do |lib|
+        libs_to_build_manually[lib] = {
+          :repo => "https://github.com/#{rebuild_org}/#{lib}",
+          :branch => dep_branch,
+        }
+      end
+
+      deps_tmp = Dir.mktmpdir("deps")
+
+      libs_to_build_manually.each do |lib, config|
+        puts "Checking out #{lib}"
+        run_command("git clone -b #{config[:branch]} #{config[:repo]} #{deps_tmp}/#{lib}", silent: false, print_command: true)
+      end
 
       puts "Starting container"
       teardown if container_exists
-      start_container(ezbake_dir)
+      start_container(deps_tmp)
 
-      puts "Installing ezbake from source"
-      run("cd /ezbake && lein install")
+      libs_to_build_manually.each do |lib, _|
+        puts "Building and installing #{lib} from source"
+        run("cd /deps/#{lib} && lein install")
+      end
 
+      fips = !ENV['FIPS'].nil?
       puts "Building openvoxdb"
       ezbake_version_var = ENV['EZBAKE_VERSION'] ? "EZBAKE_VERSION=#{ENV['EZBAKE_VERSION']}" : ''
       run("cd /code && rm -rf ruby && rm -rf output && bundle install --without test && lein install")
-      run("cd /code && COW=\"#{@debs}\" MOCK=\"#{@rpms}\" GEM_SOURCE='https://rubygems.org' #{ezbake_version_var} EZBAKE_ALLOW_UNREPRODUCIBLE_BUILDS=true EZBAKE_NODEPLOY=true LEIN_PROFILES=ezbake lein with-profile user,ezbake,provided,internal ezbake local-build")
+      run("cd /code && COW=\"#{@debs}\" MOCK=\"#{@rpms}\" GEM_SOURCE='https://rubygems.org' #{ezbake_version_var} EZBAKE_ALLOW_UNREPRODUCIBLE_BUILDS=true EZBAKE_NODEPLOY=true LEIN_PROFILES=ezbake lein with-profile #{fips ? 'fips,' : ''}user,ezbake,provided,internal ezbake local-build")
       run_command("sudo chown -R $USER output", print_command: true)
       Dir.glob('output/**/*i386*').each { |f| FileUtils.rm_rf(f) }
       Dir.glob('output/puppetdb-*.tar.gz').each { |f| FileUtils.mv(f, f.sub('puppetdb','openvoxdb'))}

tasks/upload.rake

@@ -1,6 +1,6 @@
 namespace :vox do
   desc 'Upload artifacts from the output directory to S3. Requires the AWS CLI to be installed and configured appropriately.'
-  task :upload, [:tag, :platform] do |_, args|
+  task :upload, [:platform] do |_, args|
     endpoint = ENV.fetch('ENDPOINT_URL')
     bucket = ENV.fetch('BUCKET_NAME')
     component = 'openvoxdb'
@@ -14,23 +14,31 @@ namespace :vox do
 
     abort 'You must set the ENDPOINT_URL environment variable to the S3 server you want to upload to.' if endpoint.nil? || endpoint.empty?
     abort 'You must set the BUCKET_NAME environment variable to the S3 bucket you are uploading to.' if bucket.nil? || bucket.empty?
-    abort 'You must provide a tag.' if args[:tag].nil? || args[:tag].empty?
 
-    munged_tag = args[:tag].gsub('-', '.')
     s3 = "aws s3 --endpoint-url=#{endpoint}"
 
     # Ensure the AWS CLI isn't going to fail with the given parameters
     run_command("#{s3} ls s3://#{bucket}/")
 
-    glob = "#{__dir__}/../output/**/*#{munged_tag}*"
+    config = File.expand_path("../target/staging/ezbake.rb", __dir__)
+    abort "Could not find ezbake config from the build at #{config}" unless File.exist?(config)
+    load config
+    version = EZBake::Config.fetch(:version)
+    release = EZBake::Config.fetch(:release)
+    # If release is a digit, then we built a tagged version. Otherwise,
+    # we built a snapshot and want to include that in the path to upload to.
+    tag = release =~ /^\d{1,2}$/ ? version : "#{version}-#{release}"
+
+
+    glob = "#{__dir__}/../output/**/*#{tag}*"
     if os
       # "arch" is not used here because it's all noarch
       glob += "#{os}*"
     end
     files = Dir.glob(glob)
     abort 'No files for the given tag found in the output directory.' if files.empty?
 
-    path = "s3://#{bucket}/#{component}/#{args[:tag]}"
+    path = "s3://#{bucket}/#{component}/#{tag}"
     files.each do |f|
       run_command("#{s3} cp #{f} #{path}/#{File.basename(f)}", silent: false)
     end