Feature/sandboxed workers (#21)

* Added Linux-only nsjail-based sandboxing for worker processes, including CLI support, configuration, and testing.

* Added validation for `Content-Length` in `headerPrefixPipe` and tests for oversized and negative values

* Enhanced `build.yml` to compile and install nsjail from source instead of using system package.

* Switched nsjail mode from "once" to "exec" for direct command execution with inherited stdio.

* Replaced `--time_limit` with `--rlimit_cpu` in nsjail arguments to ensure compatibility in containers without cgroupv2.

* Updated sandbox test to replace `--time_limit` with `--rlimit_cpu` and adjusted workflow to run integration tests with elevated permissions.

Author: m-messer

.github/workflows/build.yml

@@ -83,10 +83,41 @@ jobs:
 #          name: Go-results
 #          path: TestResults.json
 
+  test-sandbox:
+    name: Sandbox Integration Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: ./go.mod
+
+      - name: Install Dependencies
+        run: go mod download
+
+      - name: Install nsjail
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            autoconf bison flex g++ git \
+            libprotobuf-dev libnl-route-3-dev \
+            libtool pkg-config protobuf-compiler
+          git clone --depth=1 https://github.com/google/nsjail.git /tmp/nsjail
+          make -C /tmp/nsjail -j$(nproc)
+          sudo install -m 0755 /tmp/nsjail/nsjail /usr/sbin/nsjail
+
+      - name: Run sandbox integration tests
+        run: sudo -E go test -v -run 'TestSandboxedWorker' ./internal/execution/worker/...
+
   build_docker:
     name: Build Docker Image
     runs-on: ubuntu-latest
-    needs: [test, build]
+    needs: [test, test-sandbox, build]
     concurrency:
       group: ${{ github.ref }}
       cancel-in-progress: ${{ github.event_name == 'pull_request' || github.ref_name != github.event.repository.default_branch }}

Dockerfile

@@ -18,7 +18,64 @@ ARG COMMIT
 RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH VERSION=$VERSION COMMIT=$COMMIT \
     make build
 
-FROM scratch
+# Build nsjail from source. This stage is Linux/amd64 only; nsjail is a
+# Linux kernel feature and does not cross-compile for other OS targets.
+FROM ubuntu:24.04 AS nsjail-builder
 
-# add binary to empty scratch image
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    autoconf \
+    bison \
+    ca-certificates \
+    flex \
+    gcc \
+    g++ \
+    git \
+    libcap-dev \
+    libnl-route-3-dev \
+    libprotobuf-dev \
+    libtool \
+    make \
+    pkg-config \
+    protobuf-compiler \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN git clone --depth=1 https://github.com/google/nsjail.git /nsjail-src
+WORKDIR /nsjail-src
+RUN make -j$(nproc)
+
+# Test-only stage: golang base image (Debian bookworm) + nsjail built from source.
+# Go is pre-installed; all nsjail build deps are in Debian main — no universe needed.
+# Used by `make test-sandbox`; not referenced by the production image.
+FROM golang:1.24 AS test-sandbox
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    autoconf \
+    bison \
+    ca-certificates \
+    flex \
+    libcap-dev \
+    libnl-route-3-dev \
+    libprotobuf-dev \
+    libtool \
+    pkg-config \
+    protobuf-compiler \
+    && rm -rf /var/lib/apt/lists/*
+RUN git clone --depth=1 https://github.com/google/nsjail.git /nsjail-src && \
+    make -C /nsjail-src -j$(nproc) && \
+    cp /nsjail-src/nsjail /usr/sbin/nsjail
+
+# Runtime image. Cannot use scratch because nsjail requires shared libraries
+# (libcap, libprotobuf, libnl). Image size grows from ~8 MB to ~90-120 MB.
+# When --sandbox is not used, shimmy behaves identically to the scratch image.
+FROM ubuntu:24.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libprotobuf32t64 \
+    libnl-route-3-200 \
+    libcap2 \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=nsjail-builder /nsjail-src/nsjail /usr/sbin/nsjail
 COPY --from=builder /app/bin/shimmy /shimmy
+
+ENTRYPOINT ["/shimmy"]

Makefile

@@ -8,8 +8,9 @@ GOLDFLAGS += -X main.Commit=$(COMMIT)
 GOFLAGS = -ldflags "$(GOLDFLAGS)"
 
 BINARY_NAME ?= shimmy
+CONTAINER_ENGINE ?= docker
 
-.PHONY: all build test test-unit lcov install generate-mocks update-schema
+.PHONY: all build test test-unit test-sandbox lcov install generate-mocks update-schema
 
 all: build
 
@@ -20,6 +21,18 @@ test: test-unit
 
 test-unit:
 	go test -covermode=count -coverprofile=coverage.out ./...
+
+# Run sandbox integration tests inside a privileged container.
+# Supports Docker (default) and Podman: CONTAINER_ENGINE=podman make test-sandbox
+# On Linux with nsjail installed locally, use:
+#   go test -v -run 'TestSandboxedWorker' ./internal/execution/worker/...
+test-sandbox:
+	$(CONTAINER_ENGINE) build --target test-sandbox -t shimmy-test-sandbox .
+	$(CONTAINER_ENGINE) run --rm --privileged \
+	  -v $(shell pwd):/workspace \
+	  -w /workspace \
+	  shimmy-test-sandbox \
+	  go test -v -run 'TestSandboxedWorker' ./internal/execution/worker/...
 
 lcov:
 	gcov2lcov -infile=coverage.out -outfile=lcov.info

README.md

@@ -244,3 +244,77 @@ For example, a Wolfram Language evaluation function in `evaluation.wl` would be
 ```shell
 wolframscript -file evaluation.wl /tmp/shimmy/abc/request-data-123 /tmp/shimmy/abc/response-data-456
 ```
+
+### Sandboxed Execution (Linux only, experimental)
+
+Shimmy can wrap each worker process in an [nsjail](https://github.com/google/nsjail) sandbox to safely execute arbitrary, untrusted code. The sandbox provides:
+
+- **Filesystem confinement** — the worker can only access explicitly bind-mounted paths
+- **Resource limits** — CPU time, memory, and file descriptor caps
+- **Network isolation** — optional; disables all outbound connections
+- **Unprivileged UID** — worker runs as `nobody` (uid 65534) inside the jail
+
+Sandboxing requires Linux and the `nsjail` binary. The Docker image built from the project's `Dockerfile` includes nsjail at `/usr/sbin/nsjail`. On the host, install it with `sudo apt install nsjail` (Ubuntu 22.04+) or build from source.
+
+Enable sandboxing with `--sandbox` and configure it with the flags below:
+
+| Flag | Env var | Default | Description |
+|------|---------|---------|-------------|
+| `--sandbox` | `SANDBOX_ENABLED` | `false` | Enable nsjail sandboxing |
+| `--sandbox-nsjail-path` | `SANDBOX_NSJAIL_PATH` | `/usr/sbin/nsjail` | Path to the nsjail binary |
+| `--sandbox-ro-bind` | `SANDBOX_RO_BINDS` | — | Host path to bind-mount read-only (repeatable) |
+| `--sandbox-rw-bind` | `SANDBOX_RW_BINDS` | — | Host path to bind-mount read-write (repeatable) |
+| `--sandbox-tmpfs` | `SANDBOX_TMPFS` | — | Path inside the sandbox to mount as tmpfs (repeatable) |
+| `--sandbox-cpu-time` | `SANDBOX_CPU_TIME_LIMIT` | `0` (unlimited) | CPU time limit in seconds |
+| `--sandbox-memory-mb` | `SANDBOX_MEMORY_LIMIT` | `0` (unlimited) | Memory limit in megabytes |
+| `--sandbox-max-fds` | `SANDBOX_MAX_FDS` | `0` (nsjail default) | Maximum open file descriptors |
+| `--sandbox-disable-network` | `SANDBOX_DISABLE_NETWORK` | `false` | Disable network access inside the sandbox |
+| `--sandbox-seccomp` | `SANDBOX_SECCOMP` | `false` | Enable seccomp syscall filtering |
+
+A typical invocation for an untrusted Python worker:
+
+```shell
+shimmy -c python3 -a evaluation.py \
+  --sandbox \
+  --sandbox-ro-bind /usr \
+  --sandbox-ro-bind /lib \
+  --sandbox-ro-bind /lib64 \
+  --sandbox-rw-bind /tmp/shimmy \
+  --sandbox-cpu-time 30 \
+  --sandbox-memory-mb 256 \
+  --sandbox-disable-network
+```
+
+> **Note:** nsjail requires either root or user namespace support. In Docker, pass `--privileged` or grant `CAP_SYS_ADMIN`. In Kubernetes, configure the pod's security context accordingly.
+
+#### Testing sandboxing locally
+
+The sandbox integration tests verify actual security properties — filesystem isolation, CPU limits, network isolation, and stdio passthrough. They skip automatically if `nsjail` is not available.
+
+**On Linux with nsjail installed:**
+
+```shell
+go test -v -run 'TestSandboxedWorker' ./internal/execution/worker/...
+```
+
+**On macOS (or any platform) via Docker or Podman:**
+
+```shell
+make test-sandbox                          # Docker (default)
+CONTAINER_ENGINE=podman make test-sandbox  # Podman
+```
+
+This builds the `nsjail-builder` Dockerfile stage (the same nsjail used in production) and runs the tests inside a privileged container. Rootless Podman works fine: `--privileged` grants all capabilities within the user namespace, which is sufficient for nsjail to create its own sub-namespaces.
+
+To manually verify isolation, run the Docker image with a sandboxed worker that attempts to read a protected file:
+
+```shell
+docker run --rm --privileged \
+  -e FUNCTION_COMMAND=/bin/sh \
+  -e FUNCTION_ARGS="-c,cat /etc/shadow" \
+  -e SANDBOX_ENABLED=true \
+  -e SANDBOX_RO_BINDS="/usr:/bin:/lib:/lib64" \
+  ghcr.io/lambda-feedback/shimmy serve
+```
+
+The worker should exit with a non-zero code because `/etc` is not mounted inside the sandbox.

cmd/root.go

@@ -139,6 +139,74 @@ functions on arbitrary, serverless platforms.`
 				Value:    "127.0.0.1:7321",
 				Category: "rpc",
 			},
+			// sandbox flags
+			&cli.BoolFlag{
+				Name:     "sandbox",
+				Usage:    "enable nsjail sandboxing for worker processes (Linux only).",
+				Value:    false,
+				Category: "sandbox",
+				EnvVars:  []string{"SANDBOX_ENABLED"},
+			},
+			&cli.StringFlag{
+				Name:     "sandbox-nsjail-path",
+				Usage:    "path to the nsjail binary.",
+				Value:    "/usr/sbin/nsjail",
+				Category: "sandbox",
+				EnvVars:  []string{"SANDBOX_NSJAIL_PATH"},
+			},
+			&cli.StringSliceFlag{
+				Name:     "sandbox-ro-bind",
+				Usage:    "host path to bind-mount read-only inside the sandbox (repeatable).",
+				Category: "sandbox",
+				EnvVars:  []string{"SANDBOX_RO_BINDS"},
+			},
+			&cli.StringSliceFlag{
+				Name:     "sandbox-rw-bind",
+				Usage:    "host path to bind-mount read-write inside the sandbox (repeatable).",
+				Category: "sandbox",
+				EnvVars:  []string{"SANDBOX_RW_BINDS"},
+			},
+			&cli.StringSliceFlag{
+				Name:     "sandbox-tmpfs",
+				Usage:    "path inside the sandbox to mount as tmpfs (repeatable).",
+				Category: "sandbox",
+				EnvVars:  []string{"SANDBOX_TMPFS"},
+			},
+			&cli.IntFlag{
+				Name:     "sandbox-cpu-time",
+				Usage:    "CPU time limit in seconds for worker processes (0 = unlimited).",
+				Value:    0,
+				Category: "sandbox",
+				EnvVars:  []string{"SANDBOX_CPU_TIME_LIMIT"},
+			},
+			&cli.IntFlag{
+				Name:     "sandbox-memory-mb",
+				Usage:    "memory (address space) limit in megabytes for worker processes (0 = unlimited).",
+				Value:    0,
+				Category: "sandbox",
+				EnvVars:  []string{"SANDBOX_MEMORY_LIMIT"},
+			},
+			&cli.IntFlag{
+				Name:     "sandbox-max-fds",
+				Usage:    "maximum open file descriptors for worker processes (0 = nsjail default).",
+				Value:    0,
+				Category: "sandbox",
+				EnvVars:  []string{"SANDBOX_MAX_FDS"},
+			},
+			&cli.BoolFlag{
+				Name:     "sandbox-disable-network",
+				Usage:    "disable network access inside the sandbox.",
+				Value:    false,
+				Category: "sandbox",
+				EnvVars:  []string{"SANDBOX_DISABLE_NETWORK"},
+			},
+			&cli.BoolFlag{
+				Name:     "sandbox-seccomp",
+				Usage:    "enable seccomp syscall filtering inside the sandbox.",
+				Value:    false,
+				Category: "sandbox",
+				EnvVars:  []string{"SANDBOX_SECCOMP"},
+			},
 		},
 		Before: func(ctx *cli.Context) error {
 			// create the logger
@@ -263,6 +331,17 @@ func parseRootConfig(ctx *cli.Context) (config.Config, error) {
 		"rpc-transport-tcp-address":  "runtime.io.rpc.tcp.address",
 		"worker-send-timeout":        "runtime.send.timeout",
 		"worker-stop-timeout":        "runtime.stop.timeout",
+		// sandbox
+		"sandbox":                 "runtime.sandbox.enabled",
+		"sandbox-nsjail-path":     "runtime.sandbox.nsjail_path",
+		"sandbox-ro-bind":         "runtime.sandbox.ro_binds",
+		"sandbox-rw-bind":         "runtime.sandbox.rw_binds",
+		"sandbox-tmpfs":           "runtime.sandbox.tmpfs",
+		"sandbox-cpu-time":        "runtime.sandbox.cpu_time_limit",
+		"sandbox-memory-mb":       "runtime.sandbox.memory_limit",
+		"sandbox-max-fds":         "runtime.sandbox.max_fds",
+		"sandbox-disable-network": "runtime.sandbox.disable_network",
+		"sandbox-seccomp":         "runtime.sandbox.seccomp",
 	}
 
 	// parse config using env

internal/execution/supervisor/adapter_file.go

@@ -88,11 +88,21 @@ func (a *fileAdapter) Send(
 		return nil, fmt.Errorf("error creating temp dir: %w", err)
 	}
 
+	// allow sandboxed workers running as an unprivileged user to enter the dir
+	if err := os.Chmod(tmpPath, 0755); err != nil {
+		return nil, fmt.Errorf("error setting temp dir permissions: %w", err)
+	}
+
 	// create temp files for request and response data
 	reqFile, err := os.CreateTemp(tmpPath, "request-data-*")
 	if err != nil {
 		return nil, fmt.Errorf("error creating temp file: %w", err)
 	}
+
+	// allow sandboxed workers (running as nobody) to read the request file
+	if err := os.Chmod(reqFile.Name(), 0644); err != nil {
+		return nil, fmt.Errorf("error setting request file permissions: %w", err)
+	}
 	defer func() {
 		if err := os.Remove(reqFile.Name()); err != nil {
 			a.log.Error("failed to remove request file", zap.Error(err))
@@ -104,6 +114,11 @@ func (a *fileAdapter) Send(
 		return nil, fmt.Errorf("error creating temp file: %w", err)
 	}
 
+	// allow sandboxed workers (running as nobody) to write the response file
+	if err := os.Chmod(resFile.Name(), 0622); err != nil {
+		return nil, fmt.Errorf("error setting response file permissions: %w", err)
+	}
+
 	defer func() {
 		if err := resFile.Close(); err != nil {
 			a.log.Error("failed to close response file", zap.Error(err))

internal/execution/supervisor/adapter_rpc_pipe.go

@@ -9,6 +9,8 @@ import (
 	"sync"
 )
 
+const maxContentLength = 64 * 1024 * 1024 // 64 MB
+
 // headerPrefixPipe wraps another io.ReadWriteCloser and adds LSP-style headers
 type headerPrefixPipe struct {
 	stdio  io.ReadWriteCloser
@@ -67,6 +69,9 @@ func (h *headerPrefixPipe) Read(p []byte) (int, error) {
 		if err != nil {
 			return 0, fmt.Errorf("invalid Content-Length value: %s", parts[1])
 		}
+		if v < 0 || v > maxContentLength {
+			return 0, fmt.Errorf("Content-Length out of range: %d", v)
+		}
 		contentLength = v
 		break
 	}

internal/execution/supervisor/adapter_rpc_pipe_test.go

@@ -107,6 +107,27 @@ func TestHeaderPrefixPipe_MultipleMessages(t *testing.T) {
 	assert.Equal(t, msg2, readBuf[:n])
 }
 
+func TestHeaderPrefixPipe_RejectsOversizedContentLength(t *testing.T) {
+	buf := newRwc()
+	pipe := &headerPrefixPipe{stdio: buf}
+
+	header := fmt.Sprintf("Content-Length: %d\r\n\r\n", maxContentLength+1)
+	buf.(*rwc).Buffer.Write([]byte(header))
+
+	_, err := pipe.Read(make([]byte, 512))
+	assert.ErrorContains(t, err, "Content-Length out of range")
+}
+
+func TestHeaderPrefixPipe_RejectsNegativeContentLength(t *testing.T) {
+	buf := newRwc()
+	pipe := &headerPrefixPipe{stdio: buf}
+
+	buf.(*rwc).Buffer.Write([]byte("Content-Length: -1\r\n\r\n"))
+
+	_, err := pipe.Read(make([]byte, 512))
+	assert.ErrorContains(t, err, "Content-Length out of range")
+}
+
 func TestHeaderPrefixPipe_SkipsStrayOutputBeforeContentLength(t *testing.T) {
 	buf := newRwc()
 	pipe := &headerPrefixPipe{stdio: buf}