From 0717e31d662de455ace56b3cbf4d152c63698cb3 Mon Sep 17 00:00:00 2001 From: "Joshua M. Boniface" Date: Thu, 21 Nov 2024 13:57:42 -0500 Subject: [PATCH] Add support for arbitrary framework version detect Adds a system to build.py which can automatically determine which version of each upstream framework (.NET, NodeJS, others in the future) a given HEAD is at in the submodules, and provide that build argument into the Docker builds. This will ensure we can reliably build versions on either side of a framework version transition (i.e. both stable and master) without any manual work. For an explanation of how it works, see the comments in `build.yaml`. Future code framework updates will need the `build.yaml` updated to include the relevant (merge) commit hash and version. Also adds better logging, including this new framework version as well as a record of the Docker build commands for each build. --- build.py | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++--- build.yaml | 20 +++++++ 2 files changed, 177 insertions(+), 7 deletions(-) diff --git a/build.py b/build.py index 38237c4..fe4d4f2 100755 --- a/build.py +++ b/build.py @@ -8,6 +8,7 @@ from argparse import ArgumentParser from datetime import datetime from email.utils import format_datetime, localtime +from git import Repo from os import getenv import os.path from subprocess import run, PIPE @@ -50,6 +51,32 @@ def _determine_arch(build_type, build_arch, build_version): else: return PACKAGE_ARCH +def _determine_framework_versions(): + # Prepare repo object for this repository + this_repo = Repo(repo_root_dir) + + framework_args = dict() + + submodules = dict() + for submodule in this_repo.submodules: + if submodule.name in configurations["frameworks"].keys(): + for framework_arg in configurations["frameworks"][submodule.name].keys(): + framework_args[framework_arg] = None + for commit_hash in configurations["frameworks"][submodule.name][framework_arg].keys(): + try: + commit = submodule.module().commit(commit_hash) + if commit in submodule.module().iter_commits('HEAD'): + framework_args[framework_arg] = configurations["frameworks"][submodule.name][framework_arg][commit_hash] + except ValueError: + continue + + log(f"Determined the following framework versions based on current HEAD values:") + for k, v in framework_args.items(): + log(f" * {k}: {v}") + log("") + + return framework_args + def build_package_deb( jellyfin_version, build_type, build_arch, build_version, local=False @@ -112,9 +139,33 @@ def build_package_deb( # Use a unique docker image name for consistency imagename = f"{configurations[build_type]['imagename']}-{jellyfin_version}_{build_arch}-{build_type}-{build_version}" + # Prepare the list of build-args + build_args = list() + build_args.append(f"--build-arg PACKAGE_TYPE={os_type}") + build_args.append(f"--build-arg PACKAGE_VERSION={os_version}") + build_args.append(f"--build-arg PACKAGE_ARCH={PACKAGE_ARCH}") + build_args.append(f"--build-arg GCC_VERSION={crossgccvers}") + + # Determine framework versions + framework_versions = _determine_framework_versions() + for arg in framework_versions.keys(): + if framework_versions[arg] is not None: + build_args.append( + f"--build-arg {arg}={framework_versions[arg]}" + ) + + build_args = ' '.join(build_args) + # Build the dockerfile and packages + log( + f">>> {docker_build_cmd} {build_args} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + ) os.system( - f"{docker_build_cmd} --build-arg PACKAGE_TYPE={os_type} --build-arg PACKAGE_VERSION={os_version} --build-arg PACKAGE_ARCH={PACKAGE_ARCH} --build-arg GCC_VERSION={crossgccvers} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + f"{docker_build_cmd} {build_args} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + ) + + log( + f">>> {docker_run_cmd} --volume {repo_root_dir}:/jellyfin --volume {repo_root_dir}/out/{build_type}:/dist --env JELLYFIN_VERSION={jellyfin_version} --name {imagename} {imagename}" ) os.system( f"{docker_run_cmd} --volume {repo_root_dir}:/jellyfin --volume {repo_root_dir}/out/{build_type}:/dist --env JELLYFIN_VERSION={jellyfin_version} --name {imagename} {imagename}" @@ -148,9 +199,29 @@ def build_linux( # Set the archive type (tar-gz or zip) archivetypes = f"{configurations[build_type]['archivetypes']}" + # Prepare the list of build-args + build_args = list() + + # Determine framework versions + framework_versions = _determine_framework_versions() + for arg in framework_versions.keys(): + if framework_versions[arg] is not None: + build_args.append( + f"--build-arg {arg}={framework_versions[arg]}" + ) + + build_args = ' '.join(build_args) + # Build the dockerfile and packages + log( + f">>> {docker_build_cmd} {build_args} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + ) os.system( - f"{docker_build_cmd} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + f"{docker_build_cmd} {build_args} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + ) + + log( + f">>> {docker_run_cmd} --volume {repo_root_dir}:/jellyfin --volume {repo_root_dir}/out/{build_type}:/dist --env JELLYFIN_VERSION={jellyfin_version} --env BUILD_TYPE={build_type} --env PACKAGE_ARCH={PACKAGE_ARCH} --env DOTNET_TYPE=linux --env DOTNET_ARCH={DOTNET_ARCH} --env ARCHIVE_TYPES={archivetypes} --name {imagename} {imagename}" ) os.system( f"{docker_run_cmd} --volume {repo_root_dir}:/jellyfin --volume {repo_root_dir}/out/{build_type}:/dist --env JELLYFIN_VERSION={jellyfin_version} --env BUILD_TYPE={build_type} --env PACKAGE_ARCH={PACKAGE_ARCH} --env DOTNET_TYPE=linux --env DOTNET_ARCH={DOTNET_ARCH} --env ARCHIVE_TYPES={archivetypes} --name {imagename} {imagename}" @@ -184,9 +255,29 @@ def build_windows( # Set the archive type (tar-gz or zip) archivetypes = f"{configurations[build_type]['archivetypes']}" + # Prepare the list of build-args + build_args = list() + + # Determine framework versions + framework_versions = _determine_framework_versions() + for arg in framework_versions.keys(): + if framework_versions[arg] is not None: + build_args.append( + f"--build-arg {arg}={framework_versions[arg]}" + ) + + build_args = ' '.join(build_args) + # Build the dockerfile and packages + log( + f">>> {docker_build_cmd} {build_args} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + ) os.system( - f"{docker_build_cmd} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + f"{docker_build_cmd} {build_args} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + ) + + log( + f">>> {docker_run_cmd} --volume {repo_root_dir}:/jellyfin --volume {repo_root_dir}/out/{build_type}:/dist --env JELLYFIN_VERSION={jellyfin_version} --env BUILD_TYPE={build_type} --env PACKAGE_ARCH={PACKAGE_ARCH} --env DOTNET_TYPE=win --env DOTNET_ARCH={DOTNET_ARCH} --env ARCHIVE_TYPES={archivetypes} --name {imagename} {imagename}" ) os.system( f"{docker_run_cmd} --volume {repo_root_dir}:/jellyfin --volume {repo_root_dir}/out/{build_type}:/dist --env JELLYFIN_VERSION={jellyfin_version} --env BUILD_TYPE={build_type} --env PACKAGE_ARCH={PACKAGE_ARCH} --env DOTNET_TYPE=win --env DOTNET_ARCH={DOTNET_ARCH} --env ARCHIVE_TYPES={archivetypes} --name {imagename} {imagename}" @@ -220,9 +311,29 @@ def build_macos( # Set the archive type (tar-gz or zip) archivetypes = f"{configurations[build_type]['archivetypes']}" + # Prepare the list of build-args + build_args = list() + + # Determine framework versions + framework_versions = _determine_framework_versions() + for arg in framework_versions.keys(): + if framework_versions[arg] is not None: + build_args.append( + f"--build-arg {arg}={framework_versions[arg]}" + ) + + build_args = ' '.join(build_args) + # Build the dockerfile and packages + log( + f">>> {docker_build_cmd} {build_args} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + ) os.system( - f"{docker_build_cmd} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + f"{docker_build_cmd} {build_args} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + ) + + log( + f">>> {docker_run_cmd} --volume {repo_root_dir}:/jellyfin --volume {repo_root_dir}/out/{build_type}:/dist --env JELLYFIN_VERSION={jellyfin_version} --env BUILD_TYPE={build_type} --env PACKAGE_ARCH={PACKAGE_ARCH} --env DOTNET_TYPE=osx --env DOTNET_ARCH={DOTNET_ARCH} --env ARCHIVE_TYPES={archivetypes} --name {imagename} {imagename}" ) os.system( f"{docker_run_cmd} --volume {repo_root_dir}:/jellyfin --volume {repo_root_dir}/out/{build_type}:/dist --env JELLYFIN_VERSION={jellyfin_version} --env BUILD_TYPE={build_type} --env PACKAGE_ARCH={PACKAGE_ARCH} --env DOTNET_TYPE=osx --env DOTNET_ARCH={DOTNET_ARCH} --env ARCHIVE_TYPES={archivetypes} --name {imagename} {imagename}" @@ -251,9 +362,29 @@ def build_portable( # Set the archive type (tar-gz or zip) archivetypes = f"{configurations[build_type]['archivetypes']}" + # Prepare the list of build-args + build_args = list() + + # Determine framework versions + framework_versions = _determine_framework_versions() + for arg in framework_versions.keys(): + if framework_versions[arg] is not None: + build_args.append( + f"--build-arg {arg}={framework_versions[arg]}" + ) + + build_args = ' '.join(build_args) + # Build the dockerfile and packages + log( + f">>>{docker_build_cmd} {build_args} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + ) os.system( - f"{docker_build_cmd} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + f"{docker_build_cmd} {build_args} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + ) + + log( + f">>> {docker_run_cmd} --volume {repo_root_dir}:/jellyfin --volume {repo_root_dir}/out/{build_type}:/dist --env JELLYFIN_VERSION={jellyfin_version} --env BUILD_TYPE={build_type} --env ARCHIVE_TYPES={archivetypes} --name {imagename} {imagename}" ) os.system( f"{docker_run_cmd} --volume {repo_root_dir}:/jellyfin --volume {repo_root_dir}/out/{build_type}:/dist --env JELLYFIN_VERSION={jellyfin_version} --env BUILD_TYPE={build_type} --env ARCHIVE_TYPES={archivetypes} --name {imagename} {imagename}" @@ -326,12 +457,31 @@ def build_docker( ) log("") + # Prepare the list of build-args + build_args = list() + build_args.append(f"--build-arg PACKAGE_ARCH={PACKAGE_ARCH}") + build_args.append(f"--build-arg DOTNET_ARCH={DOTNET_ARCH}") + build_args.append(f"--build-arg QEMU_ARCH={QEMU_ARCH}") + build_args.append(f"--build-arg IMAGE_ARCH={IMAGE_ARCH}") + build_args.append(f"--build-arg TARGET_ARCH={TARGET_ARCH}") + build_args.append(f"--build-arg JELLYFIN_VERSION={jellyfin_version}") + + # Determine framework versions + framework_versions = _determine_framework_versions() + for arg in framework_versions.keys(): + if framework_versions[arg] is not None: + build_args.append( + f"--build-arg {arg}={framework_versions[arg]}" + ) + + build_args = ' '.join(build_args) + # Build the dockerfile log( - f">>> {docker_build_cmd} --build-arg PACKAGE_ARCH={PACKAGE_ARCH} --build-arg DOTNET_ARCH={DOTNET_ARCH} --build-arg QEMU_ARCH={QEMU_ARCH} --build-arg IMAGE_ARCH={IMAGE_ARCH} --build-arg TARGET_ARCH={TARGET_ARCH} --build-arg JELLYFIN_VERSION={jellyfin_version} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + f">>> {docker_build_cmd} {build_args} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" ) ret = os.system( - f"{docker_build_cmd} --build-arg PACKAGE_ARCH={PACKAGE_ARCH} --build-arg DOTNET_ARCH={DOTNET_ARCH} --build-arg QEMU_ARCH={QEMU_ARCH} --build-arg IMAGE_ARCH={IMAGE_ARCH} --build-arg TARGET_ARCH={TARGET_ARCH} --build-arg JELLYFIN_VERSION={jellyfin_version} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" + f"{docker_build_cmd} {build_args} --file {repo_root_dir}/{dockerfile} --tag {imagename} {repo_root_dir}" ) if ret > 0: exit(1) diff --git a/build.yaml b/build.yaml index 627fa5b..9e3ed13 100644 --- a/build.yaml +++ b/build.yaml @@ -1,6 +1,26 @@ --- # Build definitions for `build.py` +# Upstream Framework versions +# This section defines target commits after which a particular framework is required. +# This is used by build.py to automatically determine which framework version to use +# within the build containers based on whatever HEAD the project is at (from checkout.py). +# Target commits should be a merge commit! +# Target commits should be in order from oldest to newest! +# HOW THIS WORKS: +# For each submodule, and then for each upsteam framework version ARG to Docker... +# Provide a commit hash as a key, and a version as a value... +# If the given commit is in the commit tree of the current HEAD of the repo... +# Use the given version. Otherwise use the default. +frameworks: + jellyfin-web: + NODEJS_VERSION: + 6c0a64ef12b9eb40a7c4ee4b9d43d0a5f32f2287: 20 # Default + jellyfin-server: + DOTNET_VERSION: + 6d1abf67c36379f0b061095619147a3691841e21: 8.0 # Default + ceb850c77052c465af8422dcf152f1d1d1530457: 9.0 + # DEB packages debian: releases: