Spec generation

bits_helpers/build.py

@@ -16,6 +16,7 @@
 from bits_helpers.sl import Sapling
 from bits_helpers.scm import SCMError
 from bits_helpers.sync import remote_from_url
+from bits_helpers.script import GenerateScript
 from bits_helpers.workarea import logged_scm, updateReferenceRepoSpec, checkout_sources
 from bits_helpers.log import ProgressPrint, log_current_package
 from glob import glob
@@ -345,6 +346,13 @@ def generate_initdotsh(package, specs, architecture, post_build=False):
   # init.sh. This is useful for development off CVMFS, since we have a
   # slightly different directory hierarchy there.
   lines = [': "${BITS_ARCH_PREFIX:=%s}"' % architecture]
+
+  lines.extend([
+    'if [ -z "${WORK_DIR}" ]; then',
+    '    WORK_DIR=$(realpath "$0" | sed "s|/'"$BITS_ARCH_PREFIX"'.*||")',
+    '    export WORK_DIR',
+    'fi',
+  ])
 
   # Generate the part which sources the environment for all the dependencies.
   # We guarantee that a dependency is always sourced before the parts
@@ -1102,6 +1110,9 @@ def performPreferCheckWithTempDir(pkg, cmd):
       "build_requires": " ".join(spec["build_requires"]),
       "runtime_requires": " ".join(spec["runtime_requires"]),
     })
+    if not spec["package"].startswith("defaults-"):
+      gs = GenerateScript(spec)
+      gs.write(scriptDir, gs.generate_rpm_spec, str(spec["package"] + "_spec.sh"))
 
     # Define the environment so that it can be passed up to the
     # actual build script

bits_helpers/build_template.sh

@@ -324,6 +324,10 @@ fi
 # Last package built gets a "latest" mark.
 ln -snf $PKGVERSION-$PKGREVISION $ARCHITECTURE/$PKGNAME/latest
 
+if [ "$PKGNAME" != defaults-* ] && [ -f "$WORK_DIR/SPECS/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/${PKGNAME}_spec.sh" ]; then
+  bash -ex "$WORK_DIR/SPECS/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/${PKGNAME}_spec.sh"
+fi
+
 # Latest package built for a given devel prefix gets latest-$BUILD_FAMILY
 if [[ $BUILD_FAMILY ]]; then
   ln -snf $PKGVERSION-$PKGREVISION $ARCHITECTURE/$PKGNAME/latest-$BUILD_FAMILY

bits_helpers/download.py

@@ -54,7 +54,9 @@ def makedirs(path):
     if returncode != 0:
         raise OSError("makedirs() failed (return: %s):\n%s" % (returncode, out))
 
-def downloadUrllib2(source, destDir, work_dir, dest_filename=None):
+def downloadUrllib2(source, destDir, work_dir, dest_filename=None, cached_source=None):
+    if cached_source is not None:
+        source = cached_source
     try:
         dest = "/".join([destDir.rstrip("/"), dest_filename if dest_filename else basename(source)])
         headers={"Cache-Control": "no-cache"}
@@ -105,7 +107,9 @@ def downloadUrllib2(source, destDir, work_dir, dest_filename=None):
 #
 # which will be used to pack only a subset of the checkout.
 
-def downloadGit(source, dest, work_dir):
+def downloadGit(source, dest, work_dir, dest_filename=None, cached_source=None):
+    if cached_source is not None:
+        return downloadUrllib2(source, dest, dest_filename, cached_source)
     protocol, gitroot, args = parseGitUrl(source)
     tempdir = createTempDir(work_dir, "tmp")
 
@@ -214,7 +218,7 @@ def fixUrl(s):
             if s.endswith('?'): s=s[:-1]
     return s
 
-def downloadPip(source, dest, work_dir):
+def downloadPip(source, dest, work_dir, dest_filename=None, cached_source=None):
     # Valid PIP URL formats are
     # pip://package/version?[pip_options=downloadOptions&][pip=pip_command&][pip_package=package&]output=/tarbalname
     # pip://package/version/tarbalname
@@ -228,7 +232,7 @@ def downloadPip(source, dest, work_dir):
     for tar_name in tar_names:
       pypi_file = '%s-%s.tar.gz' % (tar_name, pkg[1].strip())
       pypi_url = 'https://pypi.io/packages/source/%s/%s/%s' % (pack[0], pack, pypi_file)
-      if downloadUrllib2(pypi_url, dest, work_dir, dest_filename=filename):
+      if downloadUrllib2(pypi_url, dest, work_dir, dest_filename=filename, cached_source=cached_source):
         return
     pack = pack + '==' + pkg[1].strip()
     pip_opts = "--no-deps --no-binary=:all:"
@@ -266,13 +270,16 @@ def downloadPip(source, dest, work_dir):
                     url=file["url"]
         if url is not None:
             debug("Found source on pypi - downloading")
-            return downloadUrllib2(url, dest, work_dir, dest_filename=filename)
+            return downloadUrllib2(url, dest, work_dir, dest_filename=filename, cached_source=cached_source)
 
     if not '--no-deps' in pip_opts: pip_opts = '--no-deps ' + pip_opts
     if not '--no-cache-dir' in pip_opts: pip_opts = '--no-cache-dir ' + pip_opts
-    comm = 'cd ' + dest + ";" + pip + ' download ' + pip_opts + ' --disable-pip-version-check -q -d . %s; [ -e %s ] || mv *.* %s; ls -l' % (pack, filename, filename)
-    error, output = getstatusoutput(comm)
-    return not error
+    if cached_source is None:
+        comm = 'cd ' + dest + ";" + pip + ' download ' + pip_opts + ' --disable-pip-version-check -q -d . %s; [ -e %s ] || mv *.* %s; ls -l' % (pack, filename, filename)
+        error, output = getstatusoutput(comm)
+        return not error
+    else:
+        return downloadUrllib2(url, dest, work_dir, dest_filename=filename, cached_source=cached_source)
 
 
 downloadHandlers = {
@@ -318,9 +325,9 @@ def download(source, dest, work_dir, cached_source=None):
   cacheDir = abspath(join(work_dir, "SOURCES/cache"))
   urlTypeRe = re.compile(r"([^:+]*)([^:]*)://.*")
 
-  match = urlTypeRe.match(cached_source or source)
+  match = urlTypeRe.match(source)
   if not match:
-    raise MalformedUrl(cached_source or source)
+    raise MalformedUrl(source)
 
   downloadHandler = downloadHandlers[match.group(1)]
 
@@ -336,7 +343,7 @@ def download(source, dest, work_dir, cached_source=None):
   realFile = join(downloadDir, filename)
   if not exists(realFile):
     debug("Trying to fetch source file: %s", cached_source or source)
-    downloadHandler(cached_source or source, downloadDir, work_dir, dest_filename=filename)
+    downloadHandler(source, downloadDir, work_dir, dest_filename=filename, cached_source=cached_source)
   if exists(realFile):
     executeWithErrorCheck("mkdir -p {dest}; cp {src} {dest}/".format(dest=dest, src=realFile), "Failed to move source")
   else:

bits_helpers/script.py

@@ -0,0 +1,103 @@
+import os
+from collections import OrderedDict
+
+class GenerateScript:
+    def __init__(self, spec: OrderedDict) -> None:
+        self.spec = spec
+
+    def write(self, scriptDir, generator, file:str):
+        with open(os.path.join(scriptDir, file), "w") as f:
+            f.write(generator())
+
+    def generate_rpm_spec(self):
+        """
+        Generates the content of a self-contained shell script that:
+        1. Writes the spec.in template
+        2. Expands it via envsubst to .spec
+        3. Builds the RPM
+        """
+        # Build the RPM spec template lines
+        spec_lines = [
+            '%define __os_install_post %{nil}',
+            '%define __spec_install_post %{nil}',
+            '%define _empty_manifest_terminate_build 0',
+            '%define _use_internal_dependency_generator 0',
+            '%define _source_payload w9.gzdio',
+            '%define _binary_payload w9.gzdio',
+            '',
+            f'Name: {self.spec["package"]}_{self.spec["version"]}_{self.spec["revision"]}_$PKGHASH',
+            'Version: $PKGVERSION',
+            'Release: $PKGREVISION',
+            'Summary: $PKG_NAME built as a part of CMS',
+            'BuildArch: %s' % os.uname().machine,
+            'License: CMS',
+        ]
+
+        # Add runtime dependencies if present
+        build_requires = self.spec.get("full_build_requires", set())
+        for dep in sorted(build_requires):
+            if dep.startswith("defaults-"):
+                continue
+            dep_clean = dep.lower().replace('-', '_')
+            dep_upper = dep_clean.upper()
+            spec_lines.append(
+                f"BuildRequires: {dep_clean}_${{{dep_upper}_VERSION}}_${{{dep_upper}_REVISION}}_${{{dep_upper}_HASH}}"
+            )
+
+        # Requires
+        full_requires = self.spec.get("full_runtime_requires", set())
+        for dep in sorted(full_requires):
+            if dep.startswith("defaults-"):
+                continue
+            dep_clean = dep.lower().replace('-', '_')
+            dep_upper = dep_clean.upper()
+            spec_lines.append(
+                f"Requires: {dep_clean}_${{{dep_upper}_VERSION}}_${{{dep_upper}_REVISION}}_${{{dep_upper}_HASH}}"
+            )
+
+        spec_lines.extend([
+            '',
+            '%description',
+            'CMS package for %s' % self.spec["package"],
+            'Built on: %s' % os.uname().nodename,
+            '',
+            '%install',
+            'cp -a $WORK_DIR/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/* %{buildroot}/',
+            'find %{buildroot} -type f -exec chmod u+w {} \\;',
+            'find %{buildroot} -type d -exec chmod u+w {} \\;',
+            '',
+            '%files',
+            '/*',
+        ])
+
+        spec_text = "\n".join(spec_lines)
+
+        # Return the shell script content as a string
+        script = f"""#!/bin/bash
+
+# 1 Create RPM build directories
+mkdir -p "$WORK_DIR/rpmbuild"/{{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS}}
+chmod -R u+w "$WORK_DIR/rpmbuild"
+
+# 2 Write literal spec.in template (variables preserved)
+SPEC_IN="$WORK_DIR/SPECS/$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/$PKGNAME.spec.in"
+cat <<'EOF' > "$SPEC_IN"
+{spec_text}
+EOF
+
+# 3 Expand variables into final .spec
+SPEC_OUT="$WORK_DIR/rpmbuild/SPECS/$PKGNAME.spec"
+envsubst < "$SPEC_IN" > "$SPEC_OUT"
+
+# 4 Load RPM environment
+source "$WORK_DIR/$ARCHITECTURE/rpm/latest/etc/profile.d/init.sh"
+
+# 5 Build the RPM
+rpmbuild -bb \\
+    --define "_topdir $WORK_DIR/rpmbuild" \\
+    --define "buildroot $WORK_DIR/rpmbuild/BUILDROOT/$PKGNAME" \\
+    "$SPEC_OUT"
+"""
+
+        return script
+

bits_helpers/sync.py

@@ -738,7 +738,7 @@ def upload_sources_to_s3(self, spec, filename, checksum) -> None:
     if not self.writeStore:
       return
     local_tarball_path = os.path.join(self.workdir, "SOURCES", "cache", checksum[0:2], checksum, filename)
-    remote_tarball_key = os.path.join("SOURCES", spec["package"], f"{checksum}.tar.gz")
+    remote_tarball_key = os.path.join("SOURCES", spec["package"], spec["version"], checksum[0:2], checksum, filename)
     debug("Uploading source tarball for %s to %s: %s", spec["package"], self.writeStore, remote_tarball_key)
     try:
       self.s3.upload_file(
@@ -749,21 +749,23 @@ def upload_sources_to_s3(self, spec, filename, checksum) -> None:
       info("Successfully uploaded source tarball for %s to S3.", spec["package"])
     except Exception as e:
       error("Failed to upload source tarball for %s to S3: %s", spec["package"], e)
-
-  def fetch_sources_from_s3(self, spec, checksum):
-    remote_source_key = os.path.join("SOURCES", spec["package"], f"{checksum}.tar.gz")
+  
+  def fetch_sources_from_s3(self, spec, checksum, filename):
+    remote_source_key = os.path.join("SOURCES", spec["package"], spec["version"], checksum[0:2], checksum, filename)
     debug("Generating download link for %s from %s: %s",
           spec["package"], self.remoteStore, remote_source_key)
     try:
+      self.s3.head_object(Bucket=self.remoteStore, Key=remote_source_key)  
       url = self.s3.generate_presigned_url(
         'get_object',
         Params={
           'Bucket': self.remoteStore,
           'Key': remote_source_key
-        },
+            },
         ExpiresIn=3600
-      )
-      debug("Generated download URL: %s", url)
+          )
+      info("Generated download URL: %s", url)
       return url
     except Exception as e:
+      debug("Failed to generate download URL for %s: %s", spec["package"], e)
       return None

bits_helpers/workarea.py

@@ -157,16 +157,17 @@ def scm_exec(command, directory=".", check=True):
     os.makedirs(source_dir, exist_ok=True)
     for patch in spec["patches"]:
       shutil.copyfile(os.path.join(spec["pkgdir"], 'patches', patch),os.path.join(source_dir, patch))
-  for s in spec["sources"]:
-    cached_source = None
-    if remote and hasattr(remote, 'fetch_sources_from_s3'):
-        cached_source = remote.fetch_sources_from_s3(spec, checksum=getUrlChecksum(s))
-
-    download(s, source_dir, work_dir, cached_source)
-
-    if cached_source is None and remote and hasattr(remote, 'upload_sources_to_s3'):
-        filename = s.rsplit("/", 1)[1]
-        remote.upload_sources_to_s3(spec, filename, checksum=getUrlChecksum(s))
+  if "sources" in spec:
+    for s in spec["sources"]:
+      filename = s.rsplit("/", 1)[1]
+      if remote and hasattr(remote, "fetch_sources_from_s3"):
+        cached_src = remote.fetch_sources_from_s3(spec, getUrlChecksum(s), filename)
+        debug("Cached source: %s" % cached_src)
+      else:
+        cached_src = None
+      download(s, source_dir, work_dir, cached_source=cached_src)
+      if remote and hasattr(remote, "upload_sources_to_s3") and cached_src is None:
+        remote.upload_sources_to_s3(spec, filename, getUrlChecksum(s))
   elif "source" not in spec:
     # There are no sources, so just create an empty SOURCEDIR.
     os.makedirs(source_dir, exist_ok=True)