diff --git a/README.md b/README.md index 8d7c01a..1f6b5da 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ hypeman rm --force --all ### Compose -`hypeman compose` applies a small declarative workload file for images, instances, restart/health settings, and ingresses. See [lib/compose/README.md](lib/compose/README.md#compose). +`hypeman compose` applies a small declarative workload file for images or Dockerfiles, instances, restart/health settings, and ingresses. See [lib/compose/README.md](lib/compose/README.md#compose). More ingress features: - Automatic certs diff --git a/go.mod b/go.mod index 1378a26..a85da66 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/knadh/koanf/providers/env v1.1.0 github.com/knadh/koanf/providers/file v1.2.1 github.com/knadh/koanf/v2 v2.3.2 + github.com/moby/patternmatcher v0.6.1 github.com/muesli/reflow v0.3.0 github.com/stretchr/testify v1.11.1 github.com/tidwall/gjson v1.18.0 diff --git a/go.sum b/go.sum index 183ee28..9359a49 100644 --- a/go.sum +++ b/go.sum @@ -113,6 +113,8 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= diff --git a/lib/compose/README.md b/lib/compose/README.md index 79c9bb8..1ff7ce9 100644 --- a/lib/compose/README.md +++ b/lib/compose/README.md @@ -2,7 +2,7 @@ ## Compose -`hypeman compose` is a lightweight way to declare a small workload for Hypeman. +`hypeman compose` is a lightweight way to declare a small workload for Hypeman from images or Dockerfiles. ```yaml @@ -69,9 +69,10 @@ All compose commands honor global output flags such as `--format json`, `--forma `up` applies the plan in order: -1. ensure referenced images exist and are ready -2. create or replace instances -3. create or replace ingresses +1. build Dockerfile services whose generated images are missing +2. ensure referenced images exist and are ready +3. create or replace instances +4. create or replace ingresses `down` deletes only instances and ingresses tagged as owned by the compose file. Images are left in place because they can be shared by normal `hypeman run` usage or other compose files. @@ -86,17 +87,42 @@ hypeman.compose.hash The hash is computed from the rendered resource spec before ownership tags are added. Re-running the same file is idempotent: matching resources are reported as unchanged, changed managed resources require `--replace`, and unmanaged resources with the same name are reported as conflicts. -### Environment Values +### Interpolation -Environment values can embed local files or environment variables: +String values can embed local files or environment variables: ```yaml +ingress: + - hostname: ${env:OTEL_COLLECTOR_VM_HOSTNAME} + target_port: 4318 + env: OTELCOL_CONFIG: ${file:otelcol.yaml} SIGNOZ_ACCESS_TOKEN: ${env:SIGNOZ_ACCESS_TOKEN} ``` -File paths are resolved relative to the compose file. Missing files or environment variables fail before any resources are applied. +File paths are resolved relative to the compose file. Loaded file contents are rendered the same way, so an `otelcol.yaml` referenced with `${file:otelcol.yaml}` can contain `${env:OTEL_COLLECTOR_VM_TOKEN}` or another `${file:...}` reference. Missing files or environment variables fail before any resources are applied. + +### Dockerfile Services + +A service can use `dockerfile` instead of `image`: + +```yaml +services: + worker: + dockerfile: ./Dockerfile + cmd: ["./worker"] + env: + CONFIG: ${file:worker.yaml} + restart: + policy: on_failure +``` + +The Dockerfile path is resolved relative to the compose file. The build context is the directory containing that Dockerfile. `compose up` creates a source archive, starts a Hypeman build, waits for the generated image to become ready, then creates the instance from that image. + +Compose generates the build image name from the compose name, service name, Dockerfile, and build context hash. Re-running the same file reuses the existing image; changing the Dockerfile or context produces a new image name and makes the managed instance require replacement. + +`image` and `dockerfile` are mutually exclusive for now. Use `image` for off-the-shelf images and `dockerfile` for Hypeman-built images. ### OTel Collector Example diff --git a/lib/compose/build.go b/lib/compose/build.go new file mode 100644 index 0000000..6893986 --- /dev/null +++ b/lib/compose/build.go @@ -0,0 +1,308 @@ +package compose + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "time" + + "github.com/kernel/hypeman-go" + "github.com/moby/patternmatcher" + "github.com/moby/patternmatcher/ignorefile" +) + +type desiredBuild struct { + Service string + Image string + Hash string + ImageRef string + DockerfilePath string + DockerfileContent string + Source []byte +} + +func (r *Runner) desiredBuildForService(serviceName string, service composeServiceSpec) (desiredBuild, error) { + dockerfilePath := service.Dockerfile + if !filepath.IsAbs(dockerfilePath) { + dockerfilePath = filepath.Join(filepath.Dir(r.file), dockerfilePath) + } + dockerfilePath, err := filepath.Abs(dockerfilePath) + if err != nil { + return desiredBuild{}, fmt.Errorf("service %q dockerfile: %w", serviceName, err) + } + dockerfileContent, err := os.ReadFile(dockerfilePath) + if err != nil { + return desiredBuild{}, fmt.Errorf("service %q dockerfile: %w", serviceName, err) + } + contextPath := filepath.Dir(dockerfilePath) + dockerignoreContent, err := readOptionalFile(filepath.Join(contextPath, ".dockerignore")) + if err != nil { + return desiredBuild{}, fmt.Errorf("service %q .dockerignore: %w", serviceName, err) + } + source, err := createSourceTarball(contextPath, dockerignoreContent) + if err != nil { + return desiredBuild{}, fmt.Errorf("service %q build context: %w", serviceName, err) + } + hash := buildHash(source, dockerfileContent, dockerignoreContent) + image := composeBuildImageName(r.spec.Name, serviceName, hash) + return desiredBuild{ + Service: serviceName, + Image: image, + Hash: hash, + DockerfilePath: dockerfilePath, + DockerfileContent: string(dockerfileContent), + Source: source, + }, nil +} + +func (r *Runner) planBuild(ctx context.Context, build desiredBuild) (Action, error) { + action := Action{ + Type: "build", + Name: build.Image, + Service: build.Service, + buildInput: &build, + } + + readyBuild, err := r.findReadyBuild(ctx, build) + if err != nil { + return Action{}, err + } + if readyBuild != nil { + action.Action = "unchanged" + action.Reason = "build already ready" + action.buildInput.ImageRef = runnableBuildImage(readyBuild) + return action, nil + } + + action.Action = "create" + action.Reason = "build missing" + return action, nil +} + +func (r *Runner) findReadyBuild(ctx context.Context, build desiredBuild) (*hypeman.Build, error) { + builds, err := r.client.Builds.List(ctx, hypeman.BuildListParams{ + Tags: composeTags(r.spec.Name, build.Service, composeResourceBuild, build.Hash), + }, r.opts...) + if err != nil { + return nil, fmt.Errorf("list builds for %s: %w", build.Image, err) + } + if builds == nil { + return nil, nil + } + var ready []hypeman.Build + for _, existing := range *builds { + if existing.Status == hypeman.BuildStatusReady && runnableBuildImage(&existing) != "" { + ready = append(ready, existing) + } + } + if len(ready) == 0 { + return nil, nil + } + sort.Slice(ready, func(i, j int) bool { + return ready[i].CreatedAt.After(ready[j].CreatedAt) + }) + return &ready[0], nil +} + +func (r *Runner) runBuild(ctx context.Context, build desiredBuild, verbose bool) (string, error) { + if verbose { + fmt.Fprintf(os.Stderr, "[build] image %s from %s\n", build.Image, build.DockerfilePath) + } + tags, err := json.Marshal(composeTags(r.spec.Name, build.Service, composeResourceBuild, build.Hash)) + if err != nil { + return "", err + } + started, err := r.client.Builds.New(ctx, hypeman.BuildNewParams{ + Source: bytes.NewReader(build.Source), + Dockerfile: hypeman.Opt(build.DockerfileContent), + Tags: hypeman.Opt(string(tags)), + }, r.opts...) + if err != nil { + return "", fmt.Errorf("start build %s: %w", build.Image, err) + } + if verbose { + fmt.Fprintf(os.Stderr, "[wait] build %s ready\n", started.ID) + } + readyBuild, err := r.waitBuildReady(ctx, started.ID) + if err != nil { + return "", err + } + imageRef := runnableBuildImage(readyBuild) + if imageRef == "" { + return "", fmt.Errorf("build %s did not report a runnable image", started.ID) + } + return imageRef, nil +} + +func (r *Runner) waitBuildReady(ctx context.Context, buildID string) (*hypeman.Build, error) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + for { + build, err := r.client.Builds.Get(ctx, buildID, r.opts...) + if err != nil { + return nil, fmt.Errorf("check build %s: %w", buildID, err) + } + switch build.Status { + case hypeman.BuildStatusReady: + return build, nil + case hypeman.BuildStatusFailed: + if build.Error != "" { + return nil, fmt.Errorf("build %s failed: %s", buildID, build.Error) + } + return nil, fmt.Errorf("build %s failed", buildID) + case hypeman.BuildStatusCancelled: + return nil, fmt.Errorf("build %s was cancelled", buildID) + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-ticker.C: + } + } +} + +func runnableBuildImage(build *hypeman.Build) string { + if build.ImageRef != "" { + return build.ImageRef + } + if build.ID != "" { + return fmt.Sprintf("docker.io/builds/%s:latest", build.ID) + } + return "" +} + +func composeBuildImageName(composeName, serviceName, hash string) string { + return fmt.Sprintf("compose/%s/%s:%s", composeName, serviceName, hash) +} + +func createSourceTarball(contextPath string, dockerignoreContent []byte) ([]byte, error) { + matcher, err := newDockerignoreMatcher(dockerignoreContent) + if err != nil { + return nil, err + } + var buf bytes.Buffer + gzWriter := gzip.NewWriter(&buf) + tarWriter := tar.NewWriter(gzWriter) + + err = filepath.Walk(contextPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(contextPath, path) + if err != nil { + return err + } + if relPath == "." { + return nil + } + relSlash := filepath.ToSlash(relPath) + base := filepath.Base(path) + if base == ".git" || base == "node_modules" || base == "__pycache__" || + base == ".venv" || base == "venv" || base == "target" || + base == ".docker" || base == ".dockerignore" { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + if matcher != nil { + ignored, err := matcher.MatchesOrParentMatches(relSlash) + if err != nil { + return err + } + if ignored { + return nil + } + } + + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + header.Name = relSlash + header.ModTime = time.Unix(0, 0) + header.AccessTime = time.Unix(0, 0) + header.ChangeTime = time.Unix(0, 0) + header.Uid = 0 + header.Gid = 0 + header.Uname = "" + header.Gname = "" + if info.Mode()&os.ModeSymlink != 0 { + linkTarget, err := os.Readlink(path) + if err != nil { + return err + } + header.Linkname = linkTarget + } + if err := tarWriter.WriteHeader(header); err != nil { + return err + } + if info.Mode().IsRegular() { + file, err := os.Open(path) + if err != nil { + return err + } + _, copyErr := io.Copy(tarWriter, file) + closeErr := file.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + } + return nil + }) + if err != nil { + return nil, err + } + if err := tarWriter.Close(); err != nil { + return nil, err + } + if err := gzWriter.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func newDockerignoreMatcher(content []byte) (*patternmatcher.PatternMatcher, error) { + if len(content) == 0 { + return nil, nil + } + patterns, err := ignorefile.ReadAll(bytes.NewReader(content)) + if err != nil { + return nil, err + } + if len(patterns) == 0 { + return nil, nil + } + return patternmatcher.New(patterns) +} + +func readOptionalFile(path string) ([]byte, error) { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return data, err +} + +func buildHash(parts ...[]byte) string { + sum := sha256.New() + for _, part := range parts { + sum.Write(part) + } + return hex.EncodeToString(sum.Sum(nil))[:12] +} diff --git a/lib/compose/compose.go b/lib/compose/compose.go index 29032d7..621c6d5 100644 --- a/lib/compose/compose.go +++ b/lib/compose/compose.go @@ -13,6 +13,7 @@ const ( composeResourceInstance = "instance" composeResourceIngress = "ingress" + composeResourceBuild = "build" ) type Runner struct { @@ -56,6 +57,7 @@ type Action struct { ingressID string instanceInput hypeman.InstanceNewParams ingressInput hypeman.IngressNewParams + buildInput *desiredBuild } func NewRunner(file string, client hypeman.Client, opts ...option.RequestOption) (*Runner, error) { diff --git a/lib/compose/compose_test.go b/lib/compose/compose_test.go index 6b0c5c2..b4dbab5 100644 --- a/lib/compose/compose_test.go +++ b/lib/compose/compose_test.go @@ -1,39 +1,59 @@ package compose import ( + "archive/tar" + "bytes" + "compress/gzip" "encoding/json" + "errors" + "io" "os" "path/filepath" "testing" + "github.com/kernel/hypeman-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestLoadComposeSpecInterpolatesFilesAndEnv(t *testing.T) { dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "otelcol.yaml"), []byte("receivers: {}\n"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "otelcol.yaml"), []byte("endpoint: https://${env:OTEL_COLLECTOR_VM_HOSTNAME}\ntoken: ${file:token.txt}\n"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "token.txt"), []byte("${env:OTEL_COLLECTOR_VM_TOKEN}"), 0644)) + t.Setenv("COMPOSE_NAME", "hypeship-otel") + t.Setenv("OTEL_IMAGE", "otel/opentelemetry-collector-contrib:0.108.0") + t.Setenv("OTELCOL_ENV_NAME", "OTELCOL_CONFIG") + t.Setenv("OTEL_COLLECTOR_VM_HOSTNAME", "otel.example.com") + t.Setenv("OTEL_COLLECTOR_VM_TOKEN", "collector-token") t.Setenv("SIGNOZ_ACCESS_TOKEN", "secret-token") composePath := filepath.Join(dir, "hypeman.compose.yaml") require.NoError(t, os.WriteFile(composePath, []byte(` version: 1 -name: hypeship-otel +name: ${env:COMPOSE_NAME} services: otelcol: - image: otel/opentelemetry-collector-contrib:0.108.0 - cmd: ["--config=env:OTELCOL_CONFIG"] + image: ${env:OTEL_IMAGE} + cmd: ["--config=env:${env:OTELCOL_ENV_NAME}"] env: OTELCOL_CONFIG: ${file:otelcol.yaml} SIGNOZ_ACCESS_TOKEN: ${env:SIGNOZ_ACCESS_TOKEN} + ingress: + - hostname: ${env:OTEL_COLLECTOR_VM_HOSTNAME} + target_port: 4318 `), 0644)) spec, err := loadComposeSpec(composePath) require.NoError(t, err) service := spec.Services["otelcol"] - assert.Equal(t, "receivers: {}\n", service.Env["OTELCOL_CONFIG"]) + assert.Equal(t, "hypeship-otel", spec.Name) + assert.Equal(t, "otel/opentelemetry-collector-contrib:0.108.0", service.Image) + assert.Equal(t, []string{"--config=env:OTELCOL_CONFIG"}, service.Cmd) + assert.Equal(t, "endpoint: https://otel.example.com\ntoken: collector-token\n", service.Env["OTELCOL_CONFIG"]) assert.Equal(t, "secret-token", service.Env["SIGNOZ_ACCESS_TOKEN"]) + require.Len(t, service.Ingress, 1) + assert.Equal(t, "otel.example.com", service.Ingress[0].Hostname) } func TestBuildComposeInstanceInputIncludesPolicyFields(t *testing.T) { @@ -113,7 +133,7 @@ func TestDesiredResourcesUseDeterministicNamesAndTags(t *testing.T) { }, } - instances, ingresses, images, err := runner.desiredResources() + _, instances, ingresses, images, err := runner.desiredResources() require.NoError(t, err) require.Equal(t, []string{"otel/opentelemetry-collector-contrib:0.108.0"}, images) @@ -141,6 +161,129 @@ func TestValidateComposeSpecRejectsInvalidNames(t *testing.T) { require.EqualError(t, err, "compose name must contain only lowercase letters, digits, and dashes") } +func TestValidateComposeSpecRejectsImageAndDockerfile(t *testing.T) { + err := validateComposeSpec(&composeSpec{ + Version: 1, + Name: "worker-stack", + Services: map[string]composeServiceSpec{ + "worker": {Image: "alpine:latest", Dockerfile: "./Dockerfile"}, + }, + }) + + require.EqualError(t, err, `service "worker" cannot include both image and dockerfile`) +} + +func TestDesiredResourcesBuildsDockerfileService(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte("FROM alpine:latest\nCOPY worker /worker\n"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "worker"), []byte("echo ok\n"), 0644)) + + composePath := filepath.Join(dir, "hypeman.compose.yaml") + require.NoError(t, os.WriteFile(composePath, []byte(` +version: 1 +name: worker-stack +services: + worker: + dockerfile: ./Dockerfile + cmd: ["./worker"] +`), 0644)) + + spec, err := loadComposeSpec(composePath) + require.NoError(t, err) + + runner := Runner{file: composePath, spec: spec} + builds, instances, _, images, err := runner.desiredResources() + require.NoError(t, err) + + require.Empty(t, images) + require.Len(t, builds, 1) + require.Len(t, instances, 1) + assert.Equal(t, "worker", builds[0].Service) + assert.Regexp(t, `^compose/worker-stack/worker:[a-f0-9]{12}$`, builds[0].Image) + assert.Equal(t, builds[0].Image, instances[0].Input.Image) + + again, _, _, _, err := runner.desiredResources() + require.NoError(t, err) + require.Equal(t, builds[0].Image, again[0].Image) + + require.NoError(t, os.WriteFile(filepath.Join(dir, ".dockerignore"), []byte("*.tmp\n"), 0644)) + dockerignoreChanged, _, _, _, err := runner.desiredResources() + require.NoError(t, err) + require.NotEqual(t, builds[0].Image, dockerignoreChanged[0].Image) + + require.NoError(t, os.Remove(filepath.Join(dir, ".dockerignore"))) + require.NoError(t, os.WriteFile(filepath.Join(dir, "worker"), []byte("echo changed\n"), 0644)) + changed, _, _, _, err := runner.desiredResources() + require.NoError(t, err) + require.NotEqual(t, builds[0].Image, changed[0].Image) +} + +func TestCreateSourceTarballHonorsDockerignore(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep\n"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "ignored.tmp"), []byte("ignored\n"), 0644)) + require.NoError(t, os.Mkdir(filepath.Join(dir, "nested"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "nested", "keep.txt"), []byte("nested\n"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "nested", "ignored.txt"), []byte("ignored\n"), 0644)) + + source, err := createSourceTarball(dir, []byte("*.tmp\nnested/*\n!nested/keep.txt\n")) + require.NoError(t, err) + + entries := sourceTarEntries(t, source) + assert.Contains(t, entries, "keep.txt") + assert.Contains(t, entries, "nested/keep.txt") + assert.NotContains(t, entries, "ignored.tmp") + assert.NotContains(t, entries, "nested/ignored.txt") +} + +func sourceTarEntries(t *testing.T, source []byte) []string { + t.Helper() + reader, err := gzip.NewReader(bytes.NewReader(source)) + require.NoError(t, err) + defer reader.Close() + + var entries []string + tarReader := tar.NewReader(reader) + for { + header, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + entries = append(entries, header.Name) + } + return entries +} + +func TestRunnableBuildImage(t *testing.T) { + assert.Equal(t, "builds/build-id", runnableBuildImage(&hypeman.Build{ + ID: "build-id", + ImageRef: "builds/build-id", + })) + assert.Equal(t, "docker.io/builds/build-id:latest", runnableBuildImage(&hypeman.Build{ + ID: "build-id", + })) +} + +func TestUpdateDesiredInstanceImageRehashesTags(t *testing.T) { + instances := []desiredInstance{{ + Service: "worker", + Input: hypeman.InstanceNewParams{ + Name: "worker-stack-worker", + Image: "compose/worker-stack/worker:original", + Tags: composeTags("worker-stack", "worker", composeResourceInstance, "old-hash"), + }, + }} + + require.NoError(t, updateDesiredInstanceImage(instances, "worker-stack", "worker", "builds/build-id")) + + assert.Equal(t, "builds/build-id", instances[0].Input.Image) + tags := instances[0].Input.Tags + assert.Equal(t, composeResourceInstance, tags[composeTagResource]) + assert.NotEqual(t, "old-hash", tags[composeTagHash]) + assert.Equal(t, instances[0].Hash, tags[composeTagHash]) +} + func TestConflictBlockers(t *testing.T) { blockers := conflictBlockers([]Action{ {Action: "create", Type: "image", Name: "alpine:latest"}, diff --git a/lib/compose/desired.go b/lib/compose/desired.go index f9b6584..ff8aa20 100644 --- a/lib/compose/desired.go +++ b/lib/compose/desired.go @@ -25,12 +25,14 @@ type desiredIngress struct { Input hypeman.IngressNewParams } -func (r *Runner) desiredResources() ([]desiredInstance, []desiredIngress, []string, error) { +func (r *Runner) desiredResources() ([]desiredBuild, []desiredInstance, []desiredIngress, []string, error) { serviceNames := make([]string, 0, len(r.spec.Services)) imageSet := map[string]struct{}{} for name, service := range r.spec.Services { serviceNames = append(serviceNames, name) - imageSet[service.Image] = struct{}{} + if service.Image != "" { + imageSet[service.Image] = struct{}{} + } } sort.Strings(serviceNames) @@ -40,15 +42,24 @@ func (r *Runner) desiredResources() ([]desiredInstance, []desiredIngress, []stri } sort.Strings(images) + var builds []desiredBuild instances := make([]desiredInstance, 0, len(serviceNames)) var ingresses []desiredIngress for _, serviceName := range serviceNames { service := r.spec.Services[serviceName] + if service.Dockerfile != "" { + build, err := r.desiredBuildForService(serviceName, service) + if err != nil { + return nil, nil, nil, nil, err + } + builds = append(builds, build) + service.Image = build.Image + } instanceName := composeInstanceName(r.spec.Name, serviceName) instanceInput := buildComposeInstanceInput(instanceName, service) instanceHash, err := shortHash(instanceInput) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } instanceInput.Tags = composeTags(r.spec.Name, serviceName, composeResourceInstance, instanceHash) instances = append(instances, desiredInstance{ @@ -63,7 +74,7 @@ func (r *Runner) desiredResources() ([]desiredInstance, []desiredIngress, []stri ingressInput := buildComposeIngressInput(instanceName, ingressName, ingressSpec) ingressHash, err := shortHash(ingressInput) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } ingressInput.Tags = composeTags(r.spec.Name, serviceName, composeResourceIngress, ingressHash) ingresses = append(ingresses, desiredIngress{ @@ -74,7 +85,7 @@ func (r *Runner) desiredResources() ([]desiredInstance, []desiredIngress, []stri }) } } - return instances, ingresses, images, nil + return builds, instances, ingresses, images, nil } func buildComposeInstanceInput(instanceName string, service composeServiceSpec) hypeman.InstanceNewParams { @@ -123,6 +134,26 @@ func buildComposeInstanceInput(instanceName string, service composeServiceSpec) return input } +func updateDesiredInstanceImage(instances []desiredInstance, composeName, serviceName, image string) error { + if image == "" { + return nil + } + for i := range instances { + if instances[i].Service != serviceName { + continue + } + instances[i].Input.Image = image + instances[i].Input.Tags = nil + hash, err := shortHash(instances[i].Input) + if err != nil { + return err + } + instances[i].Hash = hash + instances[i].Input.Tags = composeTags(composeName, serviceName, composeResourceInstance, hash) + } + return nil +} + func buildComposeRestartPolicy(restart *composeRestartSpec) hypeman.RestartPolicyParam { policy := hypeman.RestartPolicyParam{} if restart.Policy != "" { diff --git a/lib/compose/reconcile.go b/lib/compose/reconcile.go index 8e4339b..215348a 100644 --- a/lib/compose/reconcile.go +++ b/lib/compose/reconcile.go @@ -13,12 +13,24 @@ import ( ) func (r *Runner) Plan(ctx context.Context) (Plan, error) { - desiredInstances, desiredIngresses, images, err := r.desiredResources() + desiredBuilds, desiredInstances, desiredIngresses, images, err := r.desiredResources() if err != nil { return Plan{}, err } var actions []Action + for _, build := range desiredBuilds { + action, err := r.planBuild(ctx, build) + if err != nil { + return Plan{}, err + } + if action.buildInput != nil && action.buildInput.ImageRef != "" { + if err := updateDesiredInstanceImage(desiredInstances, r.spec.Name, build.Service, action.buildInput.ImageRef); err != nil { + return Plan{}, err + } + } + actions = append(actions, action) + } for _, image := range images { action, err := r.planImage(ctx, image) if err != nil { @@ -81,6 +93,11 @@ func (r *Runner) Up(ctx context.Context, opts UpOptions) (Plan, error) { if err := r.applyCreate(ctx, action, opts); err != nil { return result, err } + if action.Type == "build" { + if err := updatePlannedInstanceImage(result.Actions, r.spec.Name, action.Service, action.Name); err != nil { + return result, err + } + } case "replace": if opts.Verbose { fmt.Fprintf(os.Stderr, "[replace] %s %s\n", action.Type, action.Name) @@ -92,6 +109,15 @@ func (r *Runner) Up(ctx context.Context, opts UpOptions) (Plan, error) { if opts.Verbose { fmt.Fprintf(os.Stderr, "[skip] %s %s unchanged\n", action.Type, action.Name) } + if action.Type == "build" && action.buildInput != nil && action.buildInput.ImageRef != "" { + action.Name = action.buildInput.ImageRef + if err := updatePlannedInstanceImage(result.Actions, r.spec.Name, action.Service, action.Name); err != nil { + return result, err + } + if err := r.ensureImageReady(ctx, action.Name, opts.Verbose); err != nil { + return result, err + } + } if action.Type == "image" { if err := r.ensureImageReady(ctx, action.Name, opts.Verbose); err != nil { return result, err @@ -145,20 +171,17 @@ func (r *Runner) Down(ctx context.Context, verbose bool) (Plan, error) { Summary: summarizeComposeActions(actions), } if len(actions) == 0 { - _, desiredIngresses, _, err := r.desiredResources() - if err != nil { - return Plan{}, err - } - for _, ingress := range desiredIngresses { - result.Actions = append(result.Actions, Action{ - Action: "skip", - Type: "ingress", - Name: ingress.Name, - Service: ingress.Service, - Reason: "not found", - }) - } for serviceName := range r.spec.Services { + service := r.spec.Services[serviceName] + for i := range service.Ingress { + result.Actions = append(result.Actions, Action{ + Action: "skip", + Type: "ingress", + Name: composeIngressName(r.spec.Name, serviceName, i), + Service: serviceName, + Reason: "not found", + }) + } result.Actions = append(result.Actions, Action{ Action: "skip", Type: "instance", @@ -194,6 +217,21 @@ func (r *Runner) Down(ctx context.Context, verbose bool) (Plan, error) { func (r *Runner) applyCreate(ctx context.Context, action *Action, opts UpOptions) error { switch action.Type { + case "build": + if action.buildInput == nil { + return fmt.Errorf("build action %s missing build input", action.Name) + } + imageRef, err := r.runBuild(ctx, *action.buildInput, opts.Verbose) + if err != nil { + return err + } + if imageRef != "" { + action.Name = imageRef + } + if err := r.ensureImageReady(ctx, action.Name, opts.Verbose); err != nil { + return err + } + return nil case "image": return r.ensureImageReady(ctx, action.Name, opts.Verbose) case "instance": @@ -464,6 +502,25 @@ func summarizeComposeActions(actions []Action) Summary { return summary } +func updatePlannedInstanceImage(actions []Action, composeName, serviceName, image string) error { + if image == "" { + return nil + } + for i := range actions { + if actions[i].Type != "instance" || actions[i].Service != serviceName { + continue + } + actions[i].instanceInput.Image = image + actions[i].instanceInput.Tags = nil + hash, err := shortHash(actions[i].instanceInput) + if err != nil { + return err + } + actions[i].instanceInput.Tags = composeTags(composeName, serviceName, composeResourceInstance, hash) + } + return nil +} + func isHTTPNotFound(err error) bool { apiErr, ok := err.(*hypeman.Error) return ok && apiErr.Response != nil && apiErr.Response.StatusCode == 404 diff --git a/lib/compose/spec.go b/lib/compose/spec.go index 84cb9c5..105451a 100644 --- a/lib/compose/spec.go +++ b/lib/compose/spec.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "reflect" "regexp" "strings" @@ -18,6 +19,7 @@ type composeSpec struct { type composeServiceSpec struct { Image string `json:"image" yaml:"image"` + Dockerfile string `json:"dockerfile,omitempty" yaml:"dockerfile"` Entrypoint []string `json:"entrypoint,omitempty" yaml:"entrypoint"` Cmd []string `json:"cmd,omitempty" yaml:"cmd"` Env map[string]string `json:"env,omitempty" yaml:"env"` @@ -89,10 +91,10 @@ func loadComposeSpec(path string) (composeSpec, error) { if err := yaml.Unmarshal(data, &spec); err != nil { return composeSpec{}, fmt.Errorf("parse compose file: %w", err) } - if err := validateComposeSpec(&spec); err != nil { + if err := interpolateComposeSpec(&spec, filepath.Dir(path)); err != nil { return composeSpec{}, err } - if err := interpolateComposeSpec(&spec, filepath.Dir(path)); err != nil { + if err := validateComposeSpec(&spec); err != nil { return composeSpec{}, err } return spec, nil @@ -122,8 +124,11 @@ func validateComposeSpec(spec *composeSpec) error { if len(instanceName) > 63 { return fmt.Errorf("service %q produces instance name %q longer than 63 characters", name, instanceName) } - if service.Image == "" { - return fmt.Errorf("service %q image is required", name) + if service.Image == "" && service.Dockerfile == "" { + return fmt.Errorf("service %q image or dockerfile is required", name) + } + if service.Image != "" && service.Dockerfile != "" { + return fmt.Errorf("service %q cannot include both image and dockerfile", name) } for i, rule := range service.Ingress { if rule.Hostname == "" { @@ -143,20 +148,79 @@ func validateComposeSpec(spec *composeSpec) error { var composeInterpolationPattern = regexp.MustCompile(`\$\{(file|env):([^}]+)\}`) func interpolateComposeSpec(spec *composeSpec, baseDir string) error { - for serviceName, service := range spec.Services { - for key, value := range service.Env { - resolved, err := interpolateComposeValue(value, baseDir) - if err != nil { - return fmt.Errorf("service %q env %s: %w", serviceName, key, err) + return interpolateComposeFields(reflect.ValueOf(spec).Elem(), baseDir, "compose") +} + +func interpolateComposeFields(value reflect.Value, baseDir, path string) error { + if !value.IsValid() { + return nil + } + switch value.Kind() { + case reflect.Pointer: + if value.IsNil() { + return nil + } + return interpolateComposeFields(value.Elem(), baseDir, path) + case reflect.String: + resolved, err := interpolateComposeValue(value.String(), baseDir) + if err != nil { + return fmt.Errorf("%s: %w", path, err) + } + value.SetString(resolved) + case reflect.Struct: + valueType := value.Type() + for i := 0; i < value.NumField(); i++ { + field := valueType.Field(i) + if field.PkgPath != "" { + continue + } + name := composeYAMLFieldName(field) + if name == "" { + continue + } + if err := interpolateComposeFields(value.Field(i), baseDir, path+"."+name); err != nil { + return err } - service.Env[key] = resolved } - spec.Services[serviceName] = service + case reflect.Slice: + for i := 0; i < value.Len(); i++ { + if err := interpolateComposeFields(value.Index(i), baseDir, fmt.Sprintf("%s.%d", path, i)); err != nil { + return err + } + } + case reflect.Map: + for _, key := range value.MapKeys() { + item := reflect.New(value.Type().Elem()).Elem() + item.Set(value.MapIndex(key)) + if err := interpolateComposeFields(item, baseDir, path+"."+fmt.Sprint(key.Interface())); err != nil { + return err + } + value.SetMapIndex(key, item) + } } return nil } +func composeYAMLFieldName(field reflect.StructField) string { + tag := field.Tag.Get("yaml") + name, _, _ := strings.Cut(tag, ",") + if name == "-" { + return "" + } + if name != "" { + return name + } + return field.Name +} + func interpolateComposeValue(value, baseDir string) (string, error) { + return interpolateComposeValueDepth(value, baseDir, 0) +} + +func interpolateComposeValueDepth(value, baseDir string, depth int) (string, error) { + if depth > 16 { + return "", fmt.Errorf("interpolation depth exceeded") + } var out strings.Builder last := 0 matches := composeInterpolationPattern.FindAllStringSubmatchIndex(value, -1) @@ -180,7 +244,11 @@ func interpolateComposeValue(value, baseDir string) (string, error) { if err != nil { return "", fmt.Errorf("read file %s: %w", arg, err) } - out.Write(data) + rendered, err := interpolateComposeValueDepth(string(data), baseDir, depth+1) + if err != nil { + return "", fmt.Errorf("render file %s: %w", arg, err) + } + out.WriteString(rendered) } last = match[1] }