script/build-with-container: add support for overlay dir

The source dir (aka homedir, default /ceph) is mounted in the container
read-write. This is needed as the various ceph build scripts expect to
write things into the tree - often this is in the build directory - but
not always. This can lead to small messes and/or situations that are
confusing to debug, especially if one is jumping between distros often.
Add an option to use an overlay volume for the homedir - by default we
enable a persistent overlay with a supplied "upper dir" where files that
were written will appear. One can also enable a temporary overlay that
forgets the writes when the container exits - maybe useful when doing
experiments in 'interactive' mode.

To use this option run the command with the `--overlay=<dir>` option.
For example: `./src/script/build-with-container.py -b build.inner
--overlay-dir build.ovr`. This will create a directory
`build.ovr/content` automatically and all new files will appear there.
For example the build directory will appear at
`build.ovr/content/build.inner`.

To use the temporary overlay use a `-` as the directory name. For
example: `./src/script/build-with-container.py -b build.inner
--overlay-dir -`

Signed-off-by: John Mulligan <jmulligan@redhat.com>
(cherry picked from commit 794e3d0b25)

src/script/build-with-container.py

@@ -182,10 +182,16 @@ def _container_cmd(ctx, args, *, workdir=None, interactive=False):
     if workdir:
         cmd.append(f"--workdir={workdir}")
     cwd = pathlib.Path(".").absolute()
-    cmd += [
-        f"--volume={cwd}:{ctx.cli.homedir}:Z",
-        f"-eHOMEDIR={ctx.cli.homedir}",
-    ]
+    overlay = ctx.overlay()
+    if overlay and overlay.temporary:
+        cmd.append(f"--volume={cwd}:{ctx.cli.homedir}:O")
+    elif overlay:
+        cmd.append(
+            f"--volume={cwd}:{ctx.cli.homedir}:O,upperdir={overlay.upper},workdir={overlay.work}"
+        )
+    else:
+        cmd.append(f"--volume={cwd}:{ctx.cli.homedir}:Z")
+    cmd.append(f"-eHOMEDIR={ctx.cli.homedir}")
     if ctx.cli.build_dir:
         cmd.append(f"-eBUILD_DIR={ctx.cli.build_dir}")
     if ctx.cli.ccache_dir:
@@ -356,6 +362,29 @@ class Context:
         except DidNotExecute:
             pass
 
+    def overlay(self):
+        if not self.cli.overlay_dir:
+            return None
+        overlay = Overlay(temporary=self.cli.overlay_dir == "-")
+        if not overlay.temporary:
+            obase = pathlib.Path(self.cli.overlay_dir).resolve()
+            # you can't nest the workdir inside the upperdir at least on the
+            # version of podman I tried. But the workdir does need to be on the
+            # same FS according to the docs.  So make the workdir and the upper
+            # dir (content) siblings within the specified dir. podman doesn't
+            # have the courtesy to manage the workdir automatically when
+            # specifying upper dir.
+            overlay.upper = obase / "content"
+            overlay.work = obase / "work"
+        return overlay
+
+
+class Overlay:
+    def __init__(self, temporary=True, upper=None, work=None):
+        self.temporary = temporary
+        self.upper = upper
+        self.work = work
+
 
 class Builder:
     """Organize and manage the build steps."""
@@ -373,6 +402,8 @@ class Builder:
         if step in self._did_steps:
             log.info("step already done: %s", step)
             return
+        if not self._did_steps:
+            prepare_env_once(ctx)
         self._steps[step](ctx)
         self._did_steps.add(step)
         log.info("step done: %s", step)
@@ -395,6 +426,14 @@ class Builder:
             yield str(step), getattr(func, "__doc__", "")
 
 
+def prepare_env_once(ctx):
+    overlay = ctx.overlay()
+    if overlay and not overlay.temporary:
+        log.info("Creating overlay dirs: %s, %s", overlay.upper, overlay.work)
+        overlay.upper.mkdir(parents=True, exist_ok=True)
+        overlay.work.mkdir(parents=True, exist_ok=True)
+
+
 @Builder.set(Steps.DNF_CACHE)
 def dnf_cache_dir(ctx):
     """Set up a DNF cache directory for reuse across container builds."""
@@ -745,6 +784,15 @@ def parse_cli(build_step_names):
             " (the ceph source root)"
         ),
     )
+    parser.add_argument(
+        "--overlay-dir",
+        "-l",
+        help=(
+            "Mount the homedir as an overlay volume using the given dir"
+            "to host the overlay content and working dir. Specify '-' to"
+            "use a temporary overlay (discarding writes on container exit)"
+        ),
+    )
     parser.add_argument(
         "--ccache-dir",
         help=(