diff --git a/bits_helpers/build.py b/bits_helpers/build.py index d9211156..90bb5ea0 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -16,6 +16,7 @@ from bits_helpers.sl import Sapling from bits_helpers.scm import SCMError from bits_helpers.sync import remote_from_url +from bits_helpers.script import GenerateScript from bits_helpers.workarea import logged_scm, updateReferenceRepoSpec, checkout_sources from bits_helpers.log import ProgressPrint, log_current_package from glob import glob @@ -345,6 +346,13 @@ def generate_initdotsh(package, specs, architecture, post_build=False): # init.sh. This is useful for development off CVMFS, since we have a # slightly different directory hierarchy there. lines = [': "${BITS_ARCH_PREFIX:=%s}"' % architecture] + + lines.extend([ + 'if [ -z "${WORK_DIR}" ]; then', + ' WORK_DIR=$(realpath "$0" | sed "s|/'"$BITS_ARCH_PREFIX"'.*||")', + ' export WORK_DIR', + 'fi', + ]) # Generate the part which sources the environment for all the dependencies. # We guarantee that a dependency is always sourced before the parts @@ -1102,6 +1110,9 @@ def performPreferCheckWithTempDir(pkg, cmd): "build_requires": " ".join(spec["build_requires"]), "runtime_requires": " ".join(spec["runtime_requires"]), }) + if not spec["package"].startswith("defaults-"): + gs = GenerateScript(spec) + gs.write(scriptDir, gs.generate_rpm_spec, str(spec["package"] + "_spec.sh")) # Define the environment so that it can be passed up to the # actual build script diff --git a/bits_helpers/build_template.sh b/bits_helpers/build_template.sh index 7deab278..3accd104 100644 --- a/bits_helpers/build_template.sh +++ b/bits_helpers/build_template.sh @@ -324,6 +324,10 @@ fi # Last package built gets a "latest" mark. ln -snf $PKGVERSION-$PKGREVISION $ARCHITECTURE/$PKGNAME/latest +if [ "$PKGNAME" != defaults-* ] && [ -f "$WORK_DIR/SPECS/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/${PKGNAME}_spec.sh" ]; then + bash -ex "$WORK_DIR/SPECS/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/${PKGNAME}_spec.sh" +fi + # Latest package built for a given devel prefix gets latest-$BUILD_FAMILY if [[ $BUILD_FAMILY ]]; then ln -snf $PKGVERSION-$PKGREVISION $ARCHITECTURE/$PKGNAME/latest-$BUILD_FAMILY diff --git a/bits_helpers/download.py b/bits_helpers/download.py index 7d28e22f..dd2fb58e 100644 --- a/bits_helpers/download.py +++ b/bits_helpers/download.py @@ -54,7 +54,9 @@ def makedirs(path): if returncode != 0: raise OSError("makedirs() failed (return: %s):\n%s" % (returncode, out)) -def downloadUrllib2(source, destDir, work_dir, dest_filename=None): +def downloadUrllib2(source, destDir, work_dir, dest_filename=None, cached_source=None): + if cached_source is not None: + source = cached_source try: dest = "/".join([destDir.rstrip("/"), dest_filename if dest_filename else basename(source)]) headers={"Cache-Control": "no-cache"} @@ -105,7 +107,9 @@ def downloadUrllib2(source, destDir, work_dir, dest_filename=None): # # which will be used to pack only a subset of the checkout. -def downloadGit(source, dest, work_dir): +def downloadGit(source, dest, work_dir, dest_filename=None, cached_source=None): + if cached_source is not None: + return downloadUrllib2(source, dest, dest_filename, cached_source) protocol, gitroot, args = parseGitUrl(source) tempdir = createTempDir(work_dir, "tmp") @@ -214,7 +218,7 @@ def fixUrl(s): if s.endswith('?'): s=s[:-1] return s -def downloadPip(source, dest, work_dir): +def downloadPip(source, dest, work_dir, dest_filename=None, cached_source=None): # Valid PIP URL formats are # pip://package/version?[pip_options=downloadOptions&][pip=pip_command&][pip_package=package&]output=/tarbalname # pip://package/version/tarbalname @@ -228,7 +232,7 @@ def downloadPip(source, dest, work_dir): for tar_name in tar_names: pypi_file = '%s-%s.tar.gz' % (tar_name, pkg[1].strip()) pypi_url = 'https://pypi.io/packages/source/%s/%s/%s' % (pack[0], pack, pypi_file) - if downloadUrllib2(pypi_url, dest, work_dir, dest_filename=filename): + if downloadUrllib2(pypi_url, dest, work_dir, dest_filename=filename, cached_source=cached_source): return pack = pack + '==' + pkg[1].strip() pip_opts = "--no-deps --no-binary=:all:" @@ -266,13 +270,16 @@ def downloadPip(source, dest, work_dir): url=file["url"] if url is not None: debug("Found source on pypi - downloading") - return downloadUrllib2(url, dest, work_dir, dest_filename=filename) + return downloadUrllib2(url, dest, work_dir, dest_filename=filename, cached_source=cached_source) if not '--no-deps' in pip_opts: pip_opts = '--no-deps ' + pip_opts if not '--no-cache-dir' in pip_opts: pip_opts = '--no-cache-dir ' + pip_opts - comm = 'cd ' + dest + ";" + pip + ' download ' + pip_opts + ' --disable-pip-version-check -q -d . %s; [ -e %s ] || mv *.* %s; ls -l' % (pack, filename, filename) - error, output = getstatusoutput(comm) - return not error + if cached_source is None: + comm = 'cd ' + dest + ";" + pip + ' download ' + pip_opts + ' --disable-pip-version-check -q -d . %s; [ -e %s ] || mv *.* %s; ls -l' % (pack, filename, filename) + error, output = getstatusoutput(comm) + return not error + else: + return downloadUrllib2(url, dest, work_dir, dest_filename=filename, cached_source=cached_source) downloadHandlers = { @@ -318,9 +325,9 @@ def download(source, dest, work_dir, cached_source=None): cacheDir = abspath(join(work_dir, "SOURCES/cache")) urlTypeRe = re.compile(r"([^:+]*)([^:]*)://.*") - match = urlTypeRe.match(cached_source or source) + match = urlTypeRe.match(source) if not match: - raise MalformedUrl(cached_source or source) + raise MalformedUrl(source) downloadHandler = downloadHandlers[match.group(1)] @@ -336,7 +343,7 @@ def download(source, dest, work_dir, cached_source=None): realFile = join(downloadDir, filename) if not exists(realFile): debug("Trying to fetch source file: %s", cached_source or source) - downloadHandler(cached_source or source, downloadDir, work_dir, dest_filename=filename) + downloadHandler(source, downloadDir, work_dir, dest_filename=filename, cached_source=cached_source) if exists(realFile): executeWithErrorCheck("mkdir -p {dest}; cp {src} {dest}/".format(dest=dest, src=realFile), "Failed to move source") else: diff --git a/bits_helpers/script.py b/bits_helpers/script.py new file mode 100644 index 00000000..a17fb871 --- /dev/null +++ b/bits_helpers/script.py @@ -0,0 +1,103 @@ +import os +from collections import OrderedDict + +class GenerateScript: + def __init__(self, spec: OrderedDict) -> None: + self.spec = spec + + def write(self, scriptDir, generator, file:str): + with open(os.path.join(scriptDir, file), "w") as f: + f.write(generator()) + + def generate_rpm_spec(self): + """ + Generates the content of a self-contained shell script that: + 1. Writes the spec.in template + 2. Expands it via envsubst to .spec + 3. Builds the RPM + """ + # Build the RPM spec template lines + spec_lines = [ + '%define __os_install_post %{nil}', + '%define __spec_install_post %{nil}', + '%define _empty_manifest_terminate_build 0', + '%define _use_internal_dependency_generator 0', + '%define _source_payload w9.gzdio', + '%define _binary_payload w9.gzdio', + '', + f'Name: {self.spec["package"]}_{self.spec["version"]}_{self.spec["revision"]}_$PKGHASH', + 'Version: $PKGVERSION', + 'Release: $PKGREVISION', + 'Summary: $PKG_NAME built as a part of CMS', + 'BuildArch: %s' % os.uname().machine, + 'License: CMS', + ] + + # Add runtime dependencies if present + build_requires = self.spec.get("full_build_requires", set()) + for dep in sorted(build_requires): + if dep.startswith("defaults-"): + continue + dep_clean = dep.lower().replace('-', '_') + dep_upper = dep_clean.upper() + spec_lines.append( + f"BuildRequires: {dep_clean}_${{{dep_upper}_VERSION}}_${{{dep_upper}_REVISION}}_${{{dep_upper}_HASH}}" + ) + + # Requires + full_requires = self.spec.get("full_runtime_requires", set()) + for dep in sorted(full_requires): + if dep.startswith("defaults-"): + continue + dep_clean = dep.lower().replace('-', '_') + dep_upper = dep_clean.upper() + spec_lines.append( + f"Requires: {dep_clean}_${{{dep_upper}_VERSION}}_${{{dep_upper}_REVISION}}_${{{dep_upper}_HASH}}" + ) + + spec_lines.extend([ + '', + '%description', + 'CMS package for %s' % self.spec["package"], + 'Built on: %s' % os.uname().nodename, + '', + '%install', + 'cp -a $WORK_DIR/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/* %{buildroot}/', + 'find %{buildroot} -type f -exec chmod u+w {} \\;', + 'find %{buildroot} -type d -exec chmod u+w {} \\;', + '', + '%files', + '/*', + ]) + + spec_text = "\n".join(spec_lines) + + # Return the shell script content as a string + script = f"""#!/bin/bash + +# 1 Create RPM build directories +mkdir -p "$WORK_DIR/rpmbuild"/{{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS}} +chmod -R u+w "$WORK_DIR/rpmbuild" + +# 2 Write literal spec.in template (variables preserved) +SPEC_IN="$WORK_DIR/SPECS/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/$PKGNAME.spec.in" +cat <<'EOF' > "$SPEC_IN" +{spec_text} +EOF + +# 3 Expand variables into final .spec +SPEC_OUT="$WORK_DIR/rpmbuild/SPECS/$PKGNAME.spec" +envsubst < "$SPEC_IN" > "$SPEC_OUT" + +# 4 Load RPM environment +source "$WORK_DIR/$ARCHITECTURE/rpm/latest/etc/profile.d/init.sh" + +# 5 Build the RPM +rpmbuild -bb \\ + --define "_topdir $WORK_DIR/rpmbuild" \\ + --define "buildroot $WORK_DIR/rpmbuild/BUILDROOT/$PKGNAME" \\ + "$SPEC_OUT" +""" + + return script + \ No newline at end of file diff --git a/bits_helpers/sync.py b/bits_helpers/sync.py index fb01aaff..d49bb62d 100644 --- a/bits_helpers/sync.py +++ b/bits_helpers/sync.py @@ -738,7 +738,7 @@ def upload_sources_to_s3(self, spec, filename, checksum) -> None: if not self.writeStore: return local_tarball_path = os.path.join(self.workdir, "SOURCES", "cache", checksum[0:2], checksum, filename) - remote_tarball_key = os.path.join("SOURCES", spec["package"], f"{checksum}.tar.gz") + remote_tarball_key = os.path.join("SOURCES", spec["package"], spec["version"], checksum[0:2], checksum, filename) debug("Uploading source tarball for %s to %s: %s", spec["package"], self.writeStore, remote_tarball_key) try: self.s3.upload_file( @@ -749,21 +749,23 @@ def upload_sources_to_s3(self, spec, filename, checksum) -> None: info("Successfully uploaded source tarball for %s to S3.", spec["package"]) except Exception as e: error("Failed to upload source tarball for %s to S3: %s", spec["package"], e) - - def fetch_sources_from_s3(self, spec, checksum): - remote_source_key = os.path.join("SOURCES", spec["package"], f"{checksum}.tar.gz") + + def fetch_sources_from_s3(self, spec, checksum, filename): + remote_source_key = os.path.join("SOURCES", spec["package"], spec["version"], checksum[0:2], checksum, filename) debug("Generating download link for %s from %s: %s", spec["package"], self.remoteStore, remote_source_key) try: + self.s3.head_object(Bucket=self.remoteStore, Key=remote_source_key) url = self.s3.generate_presigned_url( 'get_object', Params={ 'Bucket': self.remoteStore, 'Key': remote_source_key - }, + }, ExpiresIn=3600 - ) - debug("Generated download URL: %s", url) + ) + info("Generated download URL: %s", url) return url except Exception as e: + debug("Failed to generate download URL for %s: %s", spec["package"], e) return None \ No newline at end of file diff --git a/bits_helpers/workarea.py b/bits_helpers/workarea.py index 5a3f36c3..1338048b 100644 --- a/bits_helpers/workarea.py +++ b/bits_helpers/workarea.py @@ -157,16 +157,17 @@ def scm_exec(command, directory=".", check=True): os.makedirs(source_dir, exist_ok=True) for patch in spec["patches"]: shutil.copyfile(os.path.join(spec["pkgdir"], 'patches', patch),os.path.join(source_dir, patch)) - for s in spec["sources"]: - cached_source = None - if remote and hasattr(remote, 'fetch_sources_from_s3'): - cached_source = remote.fetch_sources_from_s3(spec, checksum=getUrlChecksum(s)) - - download(s, source_dir, work_dir, cached_source) - - if cached_source is None and remote and hasattr(remote, 'upload_sources_to_s3'): - filename = s.rsplit("/", 1)[1] - remote.upload_sources_to_s3(spec, filename, checksum=getUrlChecksum(s)) + if "sources" in spec: + for s in spec["sources"]: + filename = s.rsplit("/", 1)[1] + if remote and hasattr(remote, "fetch_sources_from_s3"): + cached_src = remote.fetch_sources_from_s3(spec, getUrlChecksum(s), filename) + debug("Cached source: %s" % cached_src) + else: + cached_src = None + download(s, source_dir, work_dir, cached_source=cached_src) + if remote and hasattr(remote, "upload_sources_to_s3") and cached_src is None: + remote.upload_sources_to_s3(spec, filename, getUrlChecksum(s)) elif "source" not in spec: # There are no sources, so just create an empty SOURCEDIR. os.makedirs(source_dir, exist_ok=True)