diff --git a/.env.example b/.env.example index 20b0ba9..2a3dd43 100644 --- a/.env.example +++ b/.env.example @@ -13,6 +13,46 @@ TARGET_1_DIR=./uploads # TARGET_2_NAME=Processed # TARGET_2_DIR=/srv/uploads/processed +# --- S3 for Send (optional) --- +# When S3_BUCKET is set, files sent via Send are stored in S3 and recipients +# download straight from S3 via presigned URLs, bypassing this server. Leave +# unset to keep Send writing to a local target directory instead. +# S3_BUCKET=filebox-send-sandbox +# S3_KEY_PREFIX=send/ +# AWS_REGION=eu-north-1 +# Credentials for the dedicated IAM user (not a personal/SSO identity). +# Omit both when running on AWS with an attached instance/task role. +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= + +# --- Email (optional) --- +# Leave MAIL_SMTP_HOST unset to disable delivery: FileBox logs what it would +# have sent and carries on. A link origin must be set for mail to be sent. +# MAIL_FROM_ADDRESS is both the envelope sender and the From address — never a +# user's own address. The sharing user rides in Reply-To, and their name in the +# From display name ("John Doe (via FileBox)"), so SPF/DKIM stay aligned. +# MAIL_SMTP_HOST=smtp.bcc.no +# MAIL_SMTP_PORT=587 +# MAIL_SMTP_USER= +# MAIL_SMTP_PASS= +# starttls (default, port 587) | implicit (port 465) | none (local dev only) +# MAIL_SMTP_TLS=starttls +# MAIL_FROM_ADDRESS=filebox@bcc.no +# MAIL_FROM_NAME=FileBox +# Origin used in recipient links. Defaults to BASE_URL. Set it separately when +# the two differ — e.g. BASE_URL is this server (and the OAuth redirect) while a +# recipient opens the Vite dev server. Dev builds (make dev) fall back to +# http://localhost:8091; production builds fail to start if mail is on and +# neither is set, rather than mailing out links to the wrong host. +# MAIL_LINK_BASE_URL=http://localhost:8091 + +# Local dev with Mailpit (make mailpit): +# MAIL_SMTP_HOST=localhost +# MAIL_SMTP_PORT=1025 +# MAIL_SMTP_TLS=none +# MAIL_FROM_ADDRESS=filebox@localhost +# MAIL_LINK_BASE_URL=http://localhost:8091 + # --- OAuth (optional) --- # Leave every OIDC_* var unset to run in guest-only mode. # SESSION_KEY is required only when at least one provider is configured. diff --git a/Makefile b/Makefile index ea4de8b..54250b1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all build build-linux generate dev frontend frontend-dev clean +.PHONY: all build build-linux generate dev frontend frontend-dev mailpit clean all: generate frontend build @@ -29,6 +29,11 @@ frontend: frontend-dev: cd frontend && pnpm run dev +# Local SMTP catch-all for development: SMTP on :1025, web UI on :8025. +# Nothing it receives ever leaves the machine. +mailpit: + docker run --rm -p 1025:1025 -p 8025:8025 axllent/mailpit:v1.30.7 + # Clean build artifacts clean: rm -f filebox filebox-linux-amd64 diff --git a/README.md b/README.md index 152fb62..44a8581 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ A Go service that speaks the [TUS resumable upload protocol](https://tus.io/) in - Client-supplied SHA-256 verified after upload completes - Per-user upload history tracked in SQLite (duration, bandwidth, offset, status) - Multiple named upload targets, each bound to a filesystem directory +- Background ZIP64 preparation for large Send packages, with live progress and restart recovery - Strict filename validation to prevent directory-traversal attacks - Optional OAuth (OpenID Connect) sign-in with BCC Login and/or Microsoft Entra ID; falls back to guest mode when unconfigured - Goose migrations embedded in the binary, applied automatically on startup @@ -31,6 +32,10 @@ All configuration is via environment variables. | `BASE_URL` | _(empty)_ | Absolute base URL used to build TUS upload URLs and OAuth callback URLs when behind a reverse proxy (e.g. `https://upload.example.com`). | | `TARGET_N_NAME` | — | Name of upload target `N` (starting at 1). Referenced by the client via the TUS `target` metadata field. | | `TARGET_N_DIR` | — | Filesystem directory for target `N`. Must exist and be a directory. Completed uploads are moved here. | +| `S3_BUCKET` | _(empty)_ | When set, Send uploads are stored in this S3 bucket and recipients download via presigned URLs. Unset disables S3; Send then writes to a local target. See [S3 storage for Send](#s3-storage-for-send). | +| `S3_KEY_PREFIX` | `send/` | Key prefix for objects written to `S3_BUCKET`. A trailing `/` is added if missing. | +| `AWS_REGION` | — | Region of `S3_BUCKET`. Required when `S3_BUCKET` is set. | +| `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | — | Credentials for the app's IAM user. Optional when running on AWS with an attached instance/task role. | | `SESSION_KEY` | — | 32+ byte secret used for session storage. Required only when at least one OAuth provider is configured. | | `BOOTSTRAP_ADMIN_EMAIL` | — | Optional. On startup, if the `users` table is empty, seeds an admin grant for this email (all targets, admin flag). Ignored once any user has signed in. See [Bootstrapping the first admin](#bootstrapping-the-first-admin). | | `OIDC_BCC_*` / `OIDC_AZURE_*` | — | See [Authentication](#authentication). All `OIDC_*` variables are optional; OAuth is disabled when none are set. | @@ -46,6 +51,35 @@ TARGET_2_NAME=Processed TARGET_2_DIR=/srv/uploads/processed ``` +### S3 storage for Send + +Files shared through **Send** can be stored in S3 rather than on the server's own disk, so that neither the upload's final resting place nor the recipients' download traffic touches local infrastructure. Set `S3_BUCKET` (plus `AWS_REGION` and credentials) to enable it; leave `S3_BUCKET` unset and Send behaves as before, writing into a local target directory. + +How it works when enabled: + +- Uploads still arrive over TUS and are assembled in `UPLOAD_DIR/.tmp`, so resumability is unchanged. Once complete, the SHA-256 is verified **before** the transfer, and the file is then streamed to S3 (multipart for large files) and removed from the temp directory. +- Send uploads are tagged with the reserved target name `s3` instead of a configured target. This name is never a row in the `targets` table, so it can't be created, renamed, or deleted from the admin UI. +- Object keys are `/`. Namespacing by upload ID means same-named files never collide, and the key is derivable from the `uploads` row — so S3-backed shares need no extra columns. +- Generated ZIPs use `packages//artifacts/.zip`. They are streamed with 16 MiB multipart parts, so a 100 GiB non-seekable archive stays below S3's 10,000-part limit without requiring 100 GiB of local staging space. +- `GET /api/artifacts/{id}` performs all package checks (preparation, revocation, expiry, per-artifact download limit, and verification), records the access, and then responds `302` to a presigned S3 URL valid for 5 minutes. Old `/api/shares/{id}` links delegate to this policy path. The bucket itself stays entirely private. + +### Send archive policy + +All thresholds are binary GiB (`1 GiB = 2^30 bytes`), and the 100 GiB limit applies to the finished ZIP including its headers: + +- 10 files or fewer stay as individual downloads. +- More than 10 files whose combined source size is at most 100 GiB are prepared as one ZIP. If ZIP envelope bytes would cross the strict limit, the planner safely splits or leaves an otherwise-unpackable source direct. +- Above 100 GiB total, each source smaller than 10 GiB is packed into ordered ZIP parts no larger than 100 GiB, while sources of 10 GiB or more stay as individual downloads. + +Archives use ZIP64 with Store (no compression), which avoids spending CPU recompressing media and supports files over 4 GiB. Duplicate and legacy filenames are made safe and unique inside each archive. Original source objects are retained; package previews continue to list them, while only the planned ZIP/direct artifacts are downloadable. Preparation runs in the background, is restart-safe, and reports byte and percentage progress in both the sender and recipient views. Recipient email is delayed until the complete artifact set is ready. `maxDownloads` is enforced independently per downloadable artifact, so one ZIP download consumes one ZIP allowance. + +Completed files selected in the Send form are remembered by server upload ID. Reloading the page restores that exact draft selection without uploading the bytes again. Removing a restored row only removes it from the draft; it does not delete the stored source. + +Without S3, generated archives are published atomically under `UPLOAD_DIR/.archives//`. + +See [Package archive validation](docs/package-archive-validation.md) for the recorded local end-to-end scenarios, including real 90 GiB and split 96/24 GiB archive downloads and integrity checks. + + ## HTTP API ### JSON API @@ -138,6 +172,8 @@ OAuth sign-in establishes identity but **does not yet gate access**. Any visitor ## Deployment +FileBox's SQLite database and background preparation queue assume one active server process per database. Do not overlap instances during a rolling deployment or point multiple replicas at the same DB/storage paths; stop the old process before starting the replacement. + Two reference files ship in the repo: - `filebox.service` — a hardened systemd unit (`NoNewPrivileges`, `ProtectSystem=strict`, `ProtectHome=true`, explicit `ReadWritePaths`). Adjust `Environment=` lines and `ReadWritePaths=` to match your install. diff --git a/cmd/server/linkorigin.go b/cmd/server/linkorigin.go new file mode 100644 index 0000000..0bf6c77 --- /dev/null +++ b/cmd/server/linkorigin.go @@ -0,0 +1,8 @@ +//go:build !dev + +package main + +// Empty in production: guessing an origin would mail real recipients a link to +// the wrong host, and a sent mail can't be recalled. Set MAIL_LINK_BASE_URL or +// BASE_URL explicitly. +const devLinkOrigin = "" diff --git a/cmd/server/linkorigin_dev.go b/cmd/server/linkorigin_dev.go new file mode 100644 index 0000000..018f5ff --- /dev/null +++ b/cmd/server/linkorigin_dev.go @@ -0,0 +1,7 @@ +//go:build dev + +package main + +// Where a recipient opens a share link in development: the dev binary embeds no +// frontend (see embed_dev.go), so the SPA lives on the Vite dev server. +const devLinkOrigin = "http://localhost:8091" diff --git a/cmd/server/main.go b/cmd/server/main.go index dc0d663..864dfdb 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -14,6 +14,8 @@ import ( "filebox/internal/config" dbpkg "filebox/internal/db" db "filebox/internal/db/gen" + "filebox/internal/mail" + "filebox/internal/objectstore" "filebox/internal/server" "github.com/joho/godotenv" @@ -48,7 +50,7 @@ func main() { log.Fatalf("failed to create upload directory: %v", err) } - database, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") + database, err := sql.Open("sqlite", dbpkg.SQLiteDSN(dbPath)) if err != nil { log.Fatalf("failed to open database: %v", err) } @@ -90,6 +92,37 @@ func main() { log.Println("OAuth disabled (no OIDC_* env vars set) — running in guest-only mode") } + // Send uploads go to S3 when a bucket is configured, else to a local target. + objectStore, err := objectstore.NewFromEnv(context.Background()) + if err != nil { + log.Fatalf("failed to initialise S3 object store: %v", err) + } + if objectStore != nil { + log.Printf("S3 enabled for Send uploads (bucket: %s)", objectStore.Bucket()) + } else { + log.Println("S3 disabled (no S3_BUCKET set) — Send uploads use local targets") + } + + // Unconfigured mail is valid (NoopSender just logs), but a configured relay + // without BASE_URL would mail links that go nowhere — so that's fatal. + mailer, err := mail.NewFromEnv() + if err != nil { + log.Fatalf("failed to initialise mail: %v", err) + } + // Recipient links default to BASE_URL, but differ in dev: BASE_URL is this + // server's origin, while a recipient opens the Vite dev server. Dev builds + // fall back to Vite so local testing needs no configuration. + mailBaseURL := envOr("MAIL_LINK_BASE_URL", baseURL) + if mailBaseURL == "" { + mailBaseURL = devLinkOrigin // set only in dev builds + } + if mail.IsEnabled(mailer) && mailBaseURL == "" { + log.Fatalf("mail is configured but neither MAIL_LINK_BASE_URL nor BASE_URL is set — recipient links would point at the wrong host (dev builds fall back to the Vite dev server; production must set one explicitly)") + } + if mail.IsEnabled(mailer) { + log.Printf("mail: recipient links point at %s", mailBaseURL) + } + var frontendFS fs.FS if ef := embeddedFrontend(); ef != nil { if sub, err := fs.Sub(ef, "frontend_dist"); err == nil { @@ -97,7 +130,7 @@ func main() { } } - srv, err := server.New(queries, uploadDir, baseURL, frontendFS, authManager, sessionStore) + srv, err := server.New(queries, uploadDir, baseURL, mailBaseURL, frontendFS, authManager, sessionStore, objectStore, mailer) if err != nil { log.Fatalf("failed to create server: %v", err) } diff --git a/development.md b/development.md index ff97251..8013165 100644 --- a/development.md +++ b/development.md @@ -85,3 +85,79 @@ Per `CLAUDE.md`: ## Filename and path safety Filenames are validated twice: in the TUS `PreUploadCreateCallback` (so bad names are rejected before any bytes are accepted) and again immediately before the final rename. The validator rejects — rather than silently strips — any of: empty names, `.` and `..`, NUL bytes, and any path separator (`/` or `\`). Before renaming into a target, the server also recomputes the relative path with `filepath.Rel` and refuses the operation if it escapes the target directory. + +## Email + +Transactional mail (share notifications now, expiry-extension requests later) lives in `internal/mail`. Delivery is off unless `MAIL_SMTP_HOST` is set — `NewFromEnv` returns a `NoopSender` that logs the message instead of sending it, so dev and CI never put mail on the wire. + +Three headers do three different jobs, and the split is deliberate: + +- **Envelope sender** (`MAIL_FROM_ADDRESS`) — fixed service address. SPF checks this against the relay, and bounces return here, so it is never a user's own address. +- **`From:`** — the same service address, with the sharing user's name in the display part: `"John Doe (via FileBox)" `. Keeps DMARC alignment while the recipient still sees who shared. +- **`Reply-To:`** — the sharing user. A recipient hitting reply reaches a human, not an unattended mailbox. + +### Local testing with Mailpit + +```bash +make mailpit # SMTP on :1025, web UI on http://localhost:8025 +``` + +Point `.env` at it: + +```bash +MAIL_SMTP_HOST=localhost +MAIL_SMTP_PORT=1025 +MAIL_SMTP_TLS=none +MAIL_FROM_ADDRESS=filebox@localhost +MAIL_LINK_BASE_URL=http://localhost:8091 # where a recipient opens the link +``` + +`MAIL_LINK_BASE_URL` exists because the two origins differ in dev: `BASE_URL` is the Go server (and the OAuth redirect URI registered with the provider), while a recipient opens the Vite dev server on `:8091`. In production one `BASE_URL` covers both and this can stay unset. + +`make dev` builds with the `dev` tag and falls back to `http://localhost:8091` when neither is set, so local testing needs no extra config. Production builds have **no** fallback and refuse to start if mail is configured without an origin — guessing would mail real recipients a link to the wrong host, and a sent mail cannot be recalled. + +Creating a package in the UI now mails every recipient. Sending happens in a background goroutine, so package creation never blocks on the relay; the outcome lands on `package_recipients`: + +```bash +sqlite3 filebox.db "SELECT email, sent_at, send_error FROM package_recipients ORDER BY id DESC LIMIT 5;" +``` + +Or send a rendered sample without running the app: + +```bash +MAILPIT_SMTP=localhost:1025 go test ./internal/mail/ -run Mailpit -v +``` + +Mailpit captures everything and delivers nothing onward, so it is safe to point at real-looking addresses. Check both the HTML and plain-text tabs — clients that refuse HTML fall back to the text part. + +### The logo + +`frontend/public/logo-email.png` is a raster of `AppLogo.vue` (SVG does not render in any mail client), baked to `--ink` `#e7ecf5` and served unauthenticated from the app origin — in production by the embedded frontend, in dev by the Vite server, which is one more reason `MAIL_LINK_BASE_URL` points there. It is referenced as an absolute URL built by `mail.LogoURL`. + +Two deliberate properties: `alt=""` (the wordmark beside it already says FileBox, so a blocked image degrades to the wordmark rather than showing the name twice), and `width`/`height` attributes, since Outlook ignores CSS sizing. No other images are used — every one is a blockable request, and the file rows read fine without icons. + +Regenerate it after changing `AppLogo.vue`: + +```bash +# 45x52 = 2x the 23x26 display size, transparent background +"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --headless \ + --default-background-color=00000000 --window-size=45,52 \ + --screenshot=frontend/public/logo-email.png file:// +``` + +### Templates + +`internal/mail/templates/*.tmpl`, embedded via `go:embed`, one `.html.tmpl` + `.txt.tmpl` pair per mail. The HTML pair renders through `html/template`, so sender-supplied text is escaped; the text pair uses `text/template`. + +The share notification mirrors the recipient page it links to — the `.public-card` block in `PackageDownloadScreen.vue` — down to the wording (" sent you a package"), the file list with `fmtBytes` sizes, and the primary button. + +Two constraints shape the markup: + +- **Tables and inline styles.** Outlook ignores ` + + +
{{.ReasonLabel}} — they are asking for access again
+ + + + +
+ + + + + +
+ + + {{/* .public-brand */}} + + + + + {{/* .public-from + .public-pkgname */}} + + + + {{if .PackageName}} + + + + {{end}} + + {{/* .public-msg — the requester's own note */}} + {{if .Message}} + + + + {{end}} + + {{/* .pkg-meta as a key/value list */}} + + + + + {{/* .btn.btn-primary.btn-block */}} + + + + + {{/* .public-note */}} + + + + + + + + + + + + + + +
+ + + {{if .LogoURL}} + + {{end}} + + +
FileBox
+
+ {{.RequesterEmail}} asked you to reopen this package +
{{.PackageName}}
+ + +
{{.MessageHTML}}
+
+ + + + + + + + + + {{with .LimitLabel}} + + + + + {{end}} +
Why it failed{{.ReasonLabel}}
Expiry{{.ExpiredOn}}
Downloads{{.}}
+
+ + + + +
+ Review and extend +
+
+ Do nothing and the package stays exactly as it is — the link stays closed. +
+ If the button doesn't work, open this link:
+ {{.ManageURL}} +
+ Sent by FileBox. Reply to this email to reach {{.RequesterEmail}} directly. +
+
+ +
+ + diff --git a/internal/mail/templates/access_request.txt.tmpl b/internal/mail/templates/access_request.txt.tmpl new file mode 100644 index 0000000..622cb9c --- /dev/null +++ b/internal/mail/templates/access_request.txt.tmpl @@ -0,0 +1,23 @@ +{{- /* Plain-text twin of the HTML mail: same order, same wording. */ -}} +FileBox +{{.RequesterEmail}} asked you to reopen a package +{{- if .PackageName}} +{{.PackageName}} +{{- end}} +{{if .Message}} +{{.Message}} +{{end}} +{{.ReasonLabel}} when they tried to download. + + Expired {{.ExpiredOn}} +{{- with .LimitLabel}} + Downloads {{.}} +{{- end}} + +Extend the package to give them access again: +{{.ManageURL}} + +Ignoring this email leaves the package exactly as it is — the link stays closed. + +-- +Sent by FileBox. Reply to this email to reach {{.RequesterEmail}} directly. diff --git a/internal/mail/templates/download_notification.html.tmpl b/internal/mail/templates/download_notification.html.tmpl new file mode 100644 index 0000000..091af1e --- /dev/null +++ b/internal/mail/templates/download_notification.html.tmpl @@ -0,0 +1,156 @@ +{{/* + Author-facing twin of access_request.html.tmpl: same shell, same tokens. + See share_notification.html.tmpl for why the colours are hex and the layout + is tables. +*/}} + + + + + + +{{.FileLabel}} {{.DownloadedVerb}} downloaded + + + +
{{.DownloadLabel}}{{with .TotalLabel}} · {{.}}{{end}} — {{.WhenLabel}}
+ + + + +
+ + + + + +
+ + + {{/* .public-brand */}} + + + + + {{/* .public-from + .public-pkgname */}} + + + + {{if .PackageName}} + + + + {{end}} + + {{/* .public-files + .public-total — one row per file, ×N when repeated */}} + + + + + + + + {{/* .pkg-meta as a key/value list */}} + + + + + {{/* .btn.btn-primary.btn-block */}} + + + + + {{/* .public-note */}} + {{if .Exhausted}} + + + + {{end}} + + + + + + + + + + + +
+ + + {{if .LogoURL}} + + {{end}} + + +
FileBox
+
+ {{.FileLabel}} {{.DownloadedVerb}} downloaded from your package +
{{.PackageName}}
+ + {{range $i, $f := .Files}} + + + + + {{end}} + {{if .Truncated}} + + + + {{end}} +
{{$f.Name}}{{with $f.CountLabel}} {{.}}{{end}}{{$f.SizeLabel}}
… and {{.Truncated}} more downloads
+
+ + + + + +
{{.DownloadLabel}}{{.TotalLabel}}
+
+ + + + + + + + + + {{with .LimitLabel}} + + + + + {{end}} +
When{{.WhenLabel}}
Expiry{{.ExpiresOn}}
Downloads{{.}}
+
+ + + + +
+ {{if .Exhausted}}Review and extend{{else}}Manage package{{end}} +
+
+ Every file has now reached its download limit, so the link no longer works. +
+ If the button doesn't work, open this link:
+ {{.ManageURL}} +
+ Sent by FileBox because you asked to be notified when this package is downloaded.{{with .MuteURL}}
+ Stop these notifications{{end}} +
+
+ +
+ + diff --git a/internal/mail/templates/download_notification.txt.tmpl b/internal/mail/templates/download_notification.txt.tmpl new file mode 100644 index 0000000..36c046b --- /dev/null +++ b/internal/mail/templates/download_notification.txt.tmpl @@ -0,0 +1,29 @@ +{{- /* Plain-text twin of the HTML mail: same order, same wording. */ -}} +FileBox +{{.FileLabel}} {{.DownloadedVerb}} downloaded from your package +{{- if .PackageName}} +{{.PackageName}} +{{- end}} +{{range .Files}} {{.Name}} ({{.SizeLabel}}){{with .CountLabel}} {{.}}{{end}} +{{end}}{{- if .Truncated}} … and {{.Truncated}} more downloads +{{end}} {{.DownloadLabel}}{{with .TotalLabel}} · {{.}}{{end}} + + When {{.WhenLabel}} + Expires {{.ExpiresOn}} +{{- with .LimitLabel}} + Downloads {{.}} +{{- end}} +{{if .Exhausted}} +Every file has now reached its download limit, so the link no longer works. +Extend the package if the recipients still need it: +{{else}} +Manage the package: +{{end}}{{.ManageURL}} + +-- +Sent by FileBox because you asked to be notified when this package is +downloaded. +{{- with .MuteURL}} +Stop these notifications: +{{.}} +{{- end}} diff --git a/internal/mail/templates/share_notification.html.tmpl b/internal/mail/templates/share_notification.html.tmpl new file mode 100644 index 0000000..5133c7d --- /dev/null +++ b/internal/mail/templates/share_notification.html.tmpl @@ -0,0 +1,148 @@ +{{/* + Mirrors PackageDownloadScreen.vue's .public-card, so mail and page read alike. + Tables and inline styles because Outlook ignores + + +
{{.FileLabel}}{{with .TotalLabel}} · {{.}}{{end}} — {{.ExpiresOn}}
+ + + + +
+ + {{/* .public-card. Padding lives on the inner cell, not the table: + Outlook's Word engine ignores padding on . */}} +
+ + + +
+ + + {{/* .public-brand. alt is empty on purpose: the wordmark beside it + already says FileBox, so a blocked image degrades to the wordmark + alone rather than showing the name twice. */}} + + + + + {{/* .public-from + .public-pkgname */}} + + + + {{if .PackageName}} + + + + {{end}} + + {{/* .public-msg */}} + {{if .Message}} + + + + {{end}} + + {{/* .public-files + .public-total */}} + {{if .Files}} + + + + + + + {{end}} + + {{/* .btn.btn-primary.btn-block */}} + + + + + {{/* .public-note */}} + + + + + + + + + + + + + + +
+ + + {{if .LogoURL}} + + {{end}} + + +
FileBox
+
+ {{if .SenderName}}{{.SenderName}} {{.HeadlineVerb}}{{else}}{{.Headline}}{{end}} +
{{.PackageName}}
+ + +
{{.MessageHTML}}
+
+ + {{range $i, $f := .Files}} + + + + + {{end}} +
{{$f.Name}}{{$f.SizeLabel}}
+
+ + + + + +
{{.FileLabel}}{{.TotalLabel}}
+
+ + + + +
+ Download {{.FileLabel}}{{with .TotalLabel}} · {{.}}{{end}} +
+
+ Expires {{.ExpiresOn}}{{if .MaxDownloads}} · {{.MaxDownloads}} downloads allowed{{end}} +
+ If the button doesn't work, open this link:
+ {{.ShareURL}} +
+ Sent by FileBox{{if .SenderEmail}} on behalf of {{.SenderEmail}}{{end}}. Reply to this email to reach {{if .SenderName}}{{.SenderName}}{{else}}the sender{{end}} directly. +
+
+ +
+ + diff --git a/internal/mail/templates/share_notification.txt.tmpl b/internal/mail/templates/share_notification.txt.tmpl new file mode 100644 index 0000000..20916f6 --- /dev/null +++ b/internal/mail/templates/share_notification.txt.tmpl @@ -0,0 +1,23 @@ +{{- /* Plain-text twin of the HTML mail: same order, same wording. */ -}} +FileBox +{{.Headline}} +{{- if .PackageName}} +{{.PackageName}} +{{- end}} +{{if .Message}} +{{.Message}} +{{end}} +{{- if .Files}} +{{range .Files}} {{.Name}} ({{.SizeLabel}}) +{{end}} {{.FileLabel}}{{with .TotalLabel}} · {{.}}{{end}} +{{end}} +Download {{.FileLabel}}{{with .TotalLabel}} · {{.}}{{end}}: +{{.ShareURL}} + +Expires {{.ExpiresOn}}{{if .MaxDownloads}} · {{.MaxDownloads}} downloads allowed{{end}}. +After that the files are no longer available{{if not .Renewed}} and you will need to +ask for a new link{{end}}. + +-- +Sent by FileBox{{if .SenderEmail}} on behalf of {{.SenderEmail}}{{end}}. +Reply to this email to reach {{if .SenderName}}{{.SenderName}}{{else}}the sender{{end}} directly. diff --git a/internal/objectstore/s3.go b/internal/objectstore/s3.go new file mode 100644 index 0000000..5042b51 --- /dev/null +++ b/internal/objectstore/s3.go @@ -0,0 +1,186 @@ +// Package objectstore backs Send's file storage with AWS S3 instead of a local +// target directory. Send never lets the sender pick a target, so this is the +// single implicit destination. +package objectstore + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" +) + +// TargetName is the reserved `target` metadata value routing an upload to S3. +// Never a targets row, so the admin UI can't rename or delete it out from under +// the Send flow. +const TargetName = "s3" + +const defaultKeyPrefix = "send/" + +// Streaming ZIPs are non-seekable, so manager.Uploader cannot rewind and +// automatically increase its part size near S3's 10,000-part limit. At 16 MiB +// a 100 GiB archive uses at most 6,400 parts (plus modest buffered memory at the +// uploader's default concurrency). +const archiveMultipartPartSize = int64(16 << 20) + +// Client uploads to and presigns objects in a single S3 bucket. +type Client struct { + client *s3.Client + uploader *manager.Uploader + presigner *s3.PresignClient + bucket string + prefix string +} + +// NewFromEnv builds a Client from S3_BUCKET and optional S3_KEY_PREFIX (default +// "send/"), taking credentials from the AWS SDK's standard chain. Returns +// (nil, nil) when S3_BUCKET is unset, which callers read as "use local targets". +func NewFromEnv(ctx context.Context) (*Client, error) { + bucket := os.Getenv("S3_BUCKET") + if bucket == "" { + return nil, nil + } + + prefix := os.Getenv("S3_KEY_PREFIX") + if prefix == "" { + prefix = defaultKeyPrefix + } else if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return nil, fmt.Errorf("load AWS config: %w", err) + } + if cfg.Region == "" { + return nil, fmt.Errorf("S3_BUCKET is set but no AWS region is configured (set AWS_REGION)") + } + + cli := s3.NewFromConfig(cfg) + return &Client{ + client: cli, + uploader: manager.NewUploader(cli), + presigner: s3.NewPresignClient(cli), + bucket: bucket, + prefix: prefix, + }, nil +} + +// Bucket returns the configured bucket name, for logging. +func (c *Client) Bucket() string { return c.bucket } + +// Key derives the object key from the uploads row, so downloads recompute what +// the upload wrote — hence no extra schema. Namespacing by upload ID also rules +// out same-name collisions, so no tus.uniquePath equivalent is needed. +func (c *Client) Key(uploadID, filename string) string { + return c.prefix + uploadID + "/" + filename +} + +// PackageArtifactKey returns the private object key for a generated package +// artifact. The display filename deliberately stays out of the key: package +// names may change or contain awkward characters, while IDs are immutable. +func (c *Client) PackageArtifactKey(packageID, artifactID string) string { + return c.prefix + "packages/" + packageID + "/artifacts/" + artifactID + ".zip" +} + +// Upload streams the file at path into the bucket under key, switching to a +// multipart upload automatically for large files. +func (c *Client) Upload(ctx context.Context, key, path string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("open %s: %w", path, err) + } + defer f.Close() + + if _, err := c.uploader.Upload(ctx, &s3.PutObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(key), + Body: f, + }); err != nil { + return fmt.Errorf("upload to s3://%s/%s: %w", c.bucket, key, err) + } + return nil +} + +// UploadReader streams an object into S3. manager.Uploader performs multipart +// upload for large, non-seekable readers, which lets the archive worker pipe a +// ZIP directly to S3 without first allocating up to 100 GiB of local disk. +func (c *Client) UploadReader(ctx context.Context, key string, body io.Reader, contentType string) error { + in := &s3.PutObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(key), + Body: body, + } + if contentType != "" { + in.ContentType = aws.String(contentType) + } + if _, err := c.uploader.Upload(ctx, in, func(u *manager.Uploader) { + u.PartSize = archiveMultipartPartSize + }); err != nil { + return fmt.Errorf("upload to s3://%s/%s: %w", c.bucket, key, err) + } + return nil +} + +// Open returns a streaming reader for one private object. Callers must close +// it. Archive generation uses this to copy source uploads into a ZIP without +// routing the complete source through memory or local disk. +func (c *Client) Open(ctx context.Context, key string) (io.ReadCloser, error) { + out, err := c.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, fmt.Errorf("get s3://%s/%s: %w", c.bucket, key, err) + } + return out.Body, nil +} + +// Exists reports whether a private object is already present. Upload recovery +// uses this after a restart to distinguish "the S3 upload finished, but the DB +// update did not" from a genuinely missing object without downloading it. +func (c *Client) Exists(ctx context.Context, key string) (bool, error) { + _, err := c.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(key), + }) + if err == nil { + return true, nil + } + + var notFound *types.NotFound + if errors.As(err, ¬Found) { + return false, nil + } + var apiErr smithy.APIError + if errors.As(err, &apiErr) && (apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "NoSuchKey") { + return false, nil + } + return false, fmt.Errorf("head s3://%s/%s: %w", c.bucket, key, err) +} + +// PresignDownload returns a URL granting anonymous GET on key until expiry. The +// bucket stays private, so this is how a recipient reaches the bytes without the +// traffic transiting this server. filename sets Content-Disposition; interpolating +// it unquoted is safe because tus.SanitizeFilename has already run. +func (c *Client) PresignDownload(ctx context.Context, key, filename string, expiry time.Duration) (string, error) { + req, err := c.presigner.PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(key), + ResponseContentDisposition: aws.String(fmt.Sprintf("attachment; filename=%q", filename)), + }, s3.WithPresignExpires(expiry)) + if err != nil { + return "", fmt.Errorf("presign s3://%s/%s: %w", c.bucket, key, err) + } + return req.URL, nil +} diff --git a/internal/objectstore/s3_test.go b/internal/objectstore/s3_test.go new file mode 100644 index 0000000..3e52223 --- /dev/null +++ b/internal/objectstore/s3_test.go @@ -0,0 +1,14 @@ +package objectstore + +import "testing" + +func TestArchiveMultipartPartSizeSupportsHundredGiBStream(t *testing.T) { + const archiveLimit = int64(100 << 30) + parts := (archiveLimit + archiveMultipartPartSize - 1) / archiveMultipartPartSize + if parts > 10_000 { + t.Fatalf("100 GiB needs %d parts at %d bytes, exceeding S3's limit", parts, archiveMultipartPartSize) + } + if archiveMultipartPartSize < 5<<20 { + t.Fatalf("part size %d is below S3's 5 MiB multipart minimum", archiveMultipartPartSize) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 187151f..7561cc7 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1,6 +1,7 @@ package server import ( + "context" "encoding/json" "fmt" "io/fs" @@ -14,6 +15,8 @@ import ( "filebox/internal/auth" db "filebox/internal/db/gen" "filebox/internal/forms" + "filebox/internal/mail" + "filebox/internal/objectstore" "filebox/internal/tus" "github.com/tus/tusd/v2/pkg/filelocker" @@ -21,24 +24,37 @@ import ( tushandler "github.com/tus/tusd/v2/pkg/handler" ) +// sendFlowTarget is what the Send UI submits instead of a real target name, +// meaning "wherever Send files belong". Mirrors SEND_TARGET in +// PackageComposeForm.vue. +const sendFlowTarget = "send" + type Server struct { mux *http.ServeMux queries *db.Queries manager *auth.Manager sessions *auth.SessionStore baseURL string + // Origin for recipient links: usually baseURL, but separate in dev. + mailBaseURL string + store *objectstore.Client + mailer mail.Sender } // New constructs the HTTP server. The manager and sessions arguments may be // nil — in that case all auth routes return guest responses and uploads are -// tagged with "guest:" user_ids. -func New(queries *db.Queries, uploadDir string, baseURL string, frontendFS fs.FS, manager *auth.Manager, sessions *auth.SessionStore) (*Server, error) { +// tagged with "guest:" user_ids. A nil store means S3 is unconfigured and +// uploads finalize locally; a nil mailer disables delivery rather than panicking. +func New(queries *db.Queries, uploadDir string, baseURL string, mailBaseURL string, frontendFS fs.FS, manager *auth.Manager, sessions *auth.SessionStore, store *objectstore.Client, mailer mail.Sender) (*Server, error) { s := &Server{ - mux: http.NewServeMux(), - queries: queries, - manager: manager, - sessions: sessions, - baseURL: baseURL, + mux: http.NewServeMux(), + queries: queries, + manager: manager, + sessions: sessions, + baseURL: baseURL, + mailBaseURL: mailBaseURL, + store: store, + mailer: mailer, } if err := s.setupTus(uploadDir, baseURL); err != nil { @@ -82,8 +98,16 @@ func (s *Server) setupTus(uploadDir string, baseURL string) error { return err } - ep := tus.NewEventProcessor(s.queries, uploadDir, tempDir) - go ep.Run(h.UnroutedHandler) + ep := tus.NewEventProcessor(s.queries, uploadDir, tempDir, s.store) + go func() { + if err := ep.RecoverPending(context.Background()); err != nil { + log.Printf("upload storage recovery finished with errors: %v", err) + } + // Start consuming new upload events only after recovery. Both paths move + // the same temporary objects, so serialising startup prevents a live + // finalizer and recovery from promoting one row concurrently. + ep.Run(h.UnroutedHandler) + }() s.mux.Handle("/files/", http.StripPrefix("/files/", h)) return nil @@ -135,9 +159,34 @@ func (s *Server) preUploadCreate(hook tushandler.HookEvent) (tushandler.HTTPResp } newMeta["userid"] = canonical + // Only Send's symbolic target may route to S3. Empty must not: Home also + // submits empty when the user has no grants, which would divert ordinary + // uploads into the object store instead of the RawMaterial fallback. + if newMeta["target"] == sendFlowTarget { + if s.store != nil { + newMeta["target"] = objectstore.TargetName + } else if resolved, ok := s.resolveDefaultTarget(hook.Context); ok { + // No S3 configured; resolveDefaultTarget prefers a target named "send". + newMeta["target"] = resolved + } + } + return tushandler.HTTPResponse{}, tushandler.FileInfoChanges{MetaData: newMeta}, nil } +func (s *Server) resolveDefaultTarget(ctx context.Context) (string, bool) { + all, err := s.queries.ListTargets(ctx) + if err != nil || len(all) == 0 { + return "", false + } + for _, t := range all { + if strings.EqualFold(t.Name, "send") { + return t.Name, true + } + } + return all[0].Name, true +} + func (s *Server) resolveUploadUserID(hook tushandler.HookEvent) (string, error) { if s.sessions != nil { sid := cookieValue(hook.HTTPRequest.Header, auth.SessionCookieName) @@ -164,13 +213,26 @@ func (s *Server) resolveUploadUserID(hook tushandler.HookEvent) (string, error) } func (s *Server) setupAPI(uploadDir string) { - h := api.NewHandlers(s.queries) + h := api.NewHandlers(s.queries, uploadDir, s.store, s.mailer, s.mailBaseURL) s.mux.HandleFunc("GET /api/targets", h.ListTargets) s.mux.HandleFunc("GET /api/projects", h.ListProjects) s.mux.HandleFunc("GET /api/projects/{code}/suggestions", h.ProjectSuggestions) s.mux.HandleFunc("GET /api/arrangements", h.ListArrangements) s.mux.HandleFunc("GET /api/arrangements/{code}/sub-events", h.ListSubEvents) s.mux.HandleFunc("GET /api/uploads", h.ListUploads) + s.mux.HandleFunc("GET /api/shares/{id}", h.GetShare) + s.mux.HandleFunc("GET /api/artifacts/{id}", h.GetPackageArtifact) + s.mux.HandleFunc("GET /api/packages", h.ListPackagesByUser) + s.mux.HandleFunc("POST /api/packages", h.CreatePackage) + s.mux.HandleFunc("DELETE /api/packages/{id}", h.RevokePackage) + s.mux.HandleFunc("GET /api/packages/{id}/preview", h.GetPackagePreview) + s.mux.HandleFunc("POST /api/packages/{id}/verify", h.VerifyPackage) + s.mux.HandleFunc("POST /api/packages/{id}/access-request", h.RequestPackageAccess) + s.mux.HandleFunc("POST /api/packages/{id}/extend", h.ExtendPackage) + s.mux.HandleFunc("DELETE /api/packages/{id}/access-requests/{requestId}", h.DismissPackageAccessRequest) + s.mux.HandleFunc("PATCH /api/packages/{id}/notify", h.SetPackageNotify) + s.mux.HandleFunc("POST /api/notifications/mute/{token}", h.MutePackageNotifications) + h.StartPackagePreparationWorker(context.Background()) admin := api.NewAdminHandlers(s.queries, uploadDir) admin.Register(s.mux) diff --git a/internal/tus/hooks.go b/internal/tus/hooks.go index 14b5c77..f9af049 100644 --- a/internal/tus/hooks.go +++ b/internal/tus/hooks.go @@ -18,6 +18,7 @@ import ( db "filebox/internal/db/gen" "filebox/internal/forms" + "filebox/internal/objectstore" "filebox/internal/webhook" "github.com/tus/tusd/v2/pkg/handler" @@ -27,10 +28,220 @@ type EventProcessor struct { queries *db.Queries uploadDir string tempDir string + store *objectstore.Client } -func NewEventProcessor(queries *db.Queries, uploadDir, tempDir string) *EventProcessor { - return &EventProcessor{queries: queries, uploadDir: uploadDir, tempDir: tempDir} +// NewEventProcessor wires the upload event loop. A nil store means every upload +// finalizes locally; otherwise uploads tagged objectstore.TargetName go to S3. +func NewEventProcessor(queries *db.Queries, uploadDir, tempDir string, store *objectstore.Client) *EventProcessor { + return &EventProcessor{queries: queries, uploadDir: uploadDir, tempDir: tempDir, store: store} +} + +// RecoverPending resumes completed TUS uploads whose asynchronous promotion to +// local storage or S3 was interrupted by a process restart. If the temporary +// bytes still exist, normal finalization is replayed. If they do not, recovery +// checks whether the final object was already published before the process +// stopped and completes only the missing database bookkeeping. +func (ep *EventProcessor) RecoverPending(ctx context.Context) error { + uploads, err := ep.queries.ListPendingStorageUploads(ctx) + if err != nil { + return fmt.Errorf("list pending storage uploads: %w", err) + } + + var recoveryErrs []error + for _, upload := range uploads { + if err := ctx.Err(); err != nil { + recoveryErrs = append(recoveryErrs, err) + break + } + if err := ep.recoverPendingUpload(ctx, upload); err != nil { + wrapped := fmt.Errorf("upload %s: %w", upload.ID, err) + log.Printf("storage recovery: %v", wrapped) + recoveryErrs = append(recoveryErrs, wrapped) + } + } + return errors.Join(recoveryErrs...) +} + +func (ep *EventProcessor) recoverPendingUpload(ctx context.Context, upload db.Upload) error { + info := ep.recoveryFileInfo(upload) + completedAt := upload.CreatedAt + if upload.CompletedAt.Valid { + completedAt = upload.CompletedAt.Time + } + + tempPath := filepath.Join(ep.tempDir, upload.ID) + stat, err := os.Stat(tempPath) + switch { + case err == nil && !stat.Mode().IsRegular(): + return ep.failRecovery(ctx, upload.ID, "temporary upload is not a regular file") + case err == nil: + // Run synchronously so startup recovery can verify the resulting state. + ep.finalizeUpload(info, completedAt) + updated, getErr := ep.queries.GetUpload(ctx, upload.ID) + if getErr != nil { + return fmt.Errorf("read state after finalization: %w", getErr) + } + if updated.StorageStatus != "ready" { + return fmt.Errorf("finalization ended with storage status %q", updated.StorageStatus) + } + return nil + case !os.IsNotExist(err): + return fmt.Errorf("inspect temporary upload: %w", err) + } + + // The source disappeared, which can be the normal crash window after the + // final rename/upload but before MarkUploadStorageReady committed. + if upload.TargetName.Valid && upload.TargetName.String == objectstore.TargetName { + if ep.store == nil { + // This can be fixed by restoring S3 configuration, so leave the row + // pending for the next restart rather than turning it into data loss. + return errors.New("S3 destination configured but object store is unavailable") + } + key := ep.store.Key(upload.ID, upload.Filename) + exists, existsErr := ep.store.Exists(ctx, key) + if existsErr != nil { + return fmt.Errorf("check final S3 object: %w", existsErr) + } + if !exists { + return ep.failRecovery(ctx, upload.ID, "temporary upload and final S3 object are missing") + } + if err := ep.queries.MarkUploadStorageReady(ctx, upload.ID); err != nil { + return fmt.Errorf("mark recovered S3 upload ready: %w", err) + } + ep.cleanupFinalization(info, completedAt) + log.Printf("storage recovery: upload %s was already present at s3://%s/%s", upload.ID, ep.store.Bucket(), key) + return nil + } + + finalPath, found, err := ep.findRecoveredLocalFile(ctx, info, upload) + if err != nil { + return err + } + if !found { + return ep.failRecovery(ctx, upload.ID, "temporary upload and final local file are missing") + } + finalName := filepath.Base(finalPath) + if _, err := ep.queries.FinalizeUploadStorage(ctx, db.FinalizeUploadStorageParams{ + Filename: finalName, + ID: upload.ID, + }); err != nil { + return fmt.Errorf("record recovered local upload: %w", err) + } + ep.cleanupFinalization(info, completedAt) + log.Printf("storage recovery: upload %s was already present at %s", upload.ID, finalPath) + return nil +} + +// recoveryFileInfo prefers tusd's sidecar because it retains concatenation +// metadata, then overwrites the fields used for storage routing with the +// database row. The fallback makes recovery work even if tusd removed or did +// not finish writing the .info file. +func (ep *EventProcessor) recoveryFileInfo(upload db.Upload) handler.FileInfo { + info := handler.FileInfo{} + data, err := os.ReadFile(filepath.Join(ep.tempDir, upload.ID+".info")) + if err == nil { + if unmarshalErr := json.Unmarshal(data, &info); unmarshalErr != nil { + log.Printf("storage recovery: ignoring invalid info file for %s: %v", upload.ID, unmarshalErr) + info = handler.FileInfo{} + } else if info.ID != "" && info.ID != upload.ID { + log.Printf("storage recovery: ignoring mismatched info file for %s (contains %s)", upload.ID, info.ID) + info = handler.FileInfo{} + } + } else if !os.IsNotExist(err) { + log.Printf("storage recovery: cannot read info file for %s, using database metadata: %v", upload.ID, err) + } + + if info.MetaData == nil { + info.MetaData = handler.MetaData{} + } + info.ID = upload.ID + info.Size = upload.Size + info.Offset = upload.Size + info.IsPartial = false + info.MetaData["filename"] = upload.Filename + info.MetaData["userid"] = upload.UserID + info.MetaData["filetype"] = upload.ContentType.String + info.MetaData["sha256"] = upload.Sha256.String + info.MetaData["target"] = upload.TargetName.String + info.MetaData["formdata"] = upload.FormData.String + return info +} + +func (ep *EventProcessor) findRecoveredLocalFile(ctx context.Context, info handler.FileInfo, upload db.Upload) (string, bool, error) { + targetDir := filepath.Join(ep.uploadDir, "RawMaterial") + desiredName := info.MetaData["filename"] + configuredTarget := false + if upload.TargetName.Valid && upload.TargetName.String != "" { + target, err := ep.queries.GetTargetByName(ctx, upload.TargetName.String) + if err == nil { + configuredTarget = true + targetDir = target.Path + if target.FormKey.Valid && target.FormKey.String != "" { + if form, ok := forms.Get(target.FormKey.String); ok { + desiredName = forms.BuildFilename(form, parseFormData(info.MetaData["formdata"]), filepath.Ext(desiredName)) + } + } + } else if !errors.Is(err, sql.ErrNoRows) { + return "", false, fmt.Errorf("resolve local target: %w", err) + } + } + if stat, err := os.Stat(targetDir); err != nil { + if configuredTarget || !os.IsNotExist(err) { + return "", false, fmt.Errorf("storage target %s is unavailable: %w", targetDir, err) + } + return "", false, nil + } else if !stat.IsDir() { + return "", false, fmt.Errorf("storage target %s is not a directory", targetDir) + } + + candidates := []string{upload.Filename} + if sanitized, err := SanitizeFilename(desiredName); err == nil && sanitized != upload.Filename { + candidates = append(candidates, sanitized) + } + var match string + for _, name := range candidates { + if name == "" || name == "." || name == ".." || filepath.Base(name) != name { + continue + } + candidate := filepath.Join(targetDir, name) + stat, err := os.Stat(candidate) + if os.IsNotExist(err) { + continue + } + if err != nil { + return "", false, fmt.Errorf("inspect possible final file %s: %w", candidate, err) + } + if !stat.Mode().IsRegular() || stat.Size() != upload.Size { + continue + } + if upload.Sha256.Valid && upload.Sha256.String != "" { + actual, err := computeFileSHA256(candidate) + if err != nil { + return "", false, fmt.Errorf("verify possible final file %s: %w", candidate, err) + } + if actual != upload.Sha256.String { + continue + } + } else if upload.CompletedAt.Valid && stat.ModTime().Before(upload.CompletedAt.Time.Add(-5*time.Second)) { + // With no content hash, do not mistake an older same-name file for + // the upload that vanished from the temporary directory. + continue + } + if match != "" && match != candidate { + return "", false, errors.New("multiple possible final local files found") + } + match = candidate + } + return match, match != "", nil +} + +func (ep *EventProcessor) failRecovery(ctx context.Context, uploadID, reason string) error { + if err := ep.queries.FailUpload(ctx, uploadID); err != nil { + return fmt.Errorf("%s; mark storage failed: %w", reason, err) + } + _ = os.Remove(filepath.Join(ep.tempDir, uploadID+".info")) + return errors.New(reason) } // Run processes all tus events in a single goroutine to avoid race conditions. @@ -78,7 +289,7 @@ func (ep *EventProcessor) handleCreated(event handler.HookEvent) { targetName := info.MetaData["target"] formData := info.MetaData["formdata"] - err := ep.queries.CreateUpload(context.Background(), db.CreateUploadParams{ + err := ep.queries.CreatePendingUpload(context.Background(), db.CreatePendingUploadParams{ ID: info.ID, UserID: userID, Filename: filename, @@ -144,7 +355,118 @@ func (ep *EventProcessor) handleComplete(event handler.HookEvent) { go ep.finalizeUpload(info, completedAt) } +// finalizeUpload moves a completed upload out of the temp area into its final +// home, then does the bookkeeping common to every destination. func (ep *EventProcessor) finalizeUpload(info handler.FileInfo, completedAt time.Time) { + var storedFilename string + stored := false + if ep.store != nil && info.MetaData["target"] == objectstore.TargetName { + storedFilename, stored = ep.storeToS3(info) + } else { + storedFilename, stored = ep.storeToDisk(info, completedAt) + } + if stored { + if _, err := ep.queries.FinalizeUploadStorage(context.Background(), db.FinalizeUploadStorageParams{ + Filename: storedFilename, + ID: info.ID, + }); err != nil { + // Keep tusd's sidecar for restart reconciliation. The published bytes + // remain inaccessible to package preparation until this atomic state + // transition succeeds. + log.Printf("warning: failed to record final storage for upload %s: %v", info.ID, err) + return + } + } + ep.cleanupFinalization(info, completedAt) +} + +// cleanupFinalization removes tusd's temporary bookkeeping after either normal +// finalization or restart recovery, including concatenation partials. +func (ep *EventProcessor) cleanupFinalization(info handler.FileInfo, completedAt time.Time) { + // For concatenated uploads, fix the duration to measure from the earliest + // partial upload's creation time (the final upload is created and completed + // in the same request, so its created_at == completed_at). + if info.PartialUploads != nil { + var earliest time.Time + for _, partialID := range info.PartialUploads { + p, err := ep.queries.GetUpload(context.Background(), partialID) + if err != nil { + continue + } + if earliest.IsZero() || p.CreatedAt.Before(earliest) { + earliest = p.CreatedAt + } + } + if !earliest.IsZero() { + durationMs := completedAt.Sub(earliest).Milliseconds() + ep.queries.UpdateDurationMs(context.Background(), db.UpdateDurationMsParams{ + DurationMs: sql.NullInt64{Int64: durationMs, Valid: true}, + ID: info.ID, + }) + } + + // Clean up partial files and .info files + for _, partialID := range info.PartialUploads { + os.Remove(filepath.Join(ep.tempDir, partialID)) + os.Remove(filepath.Join(ep.tempDir, partialID+".info")) + } + // Delete partial DB records + for _, partialID := range info.PartialUploads { + ep.queries.DeleteUpload(context.Background(), partialID) + } + } + + // Remove the .info file for the completed upload + os.Remove(filepath.Join(ep.tempDir, info.ID+".info")) +} + +// storeToS3 promotes a Send upload into the object store, verifying the SHA-256 +// before transferring (unlike the local path) so a corrupt file costs no +// bandwidth. No form handling: an S3-bound upload can't have one. +func (ep *EventProcessor) storeToS3(info handler.FileInfo) (string, bool) { + srcPath := filepath.Join(ep.tempDir, info.ID) + + filename, err := SanitizeFilename(info.MetaData["filename"]) + if err != nil { + log.Printf("rejecting upload %s: %v", info.ID, err) + ep.queries.FailUpload(context.Background(), info.ID) + os.Remove(srcPath) + return "", false + } + + if expected := info.MetaData["sha256"]; expected != "" { + actual, err := computeFileSHA256(srcPath) + if err != nil { + log.Printf("error computing SHA-256 for %s: %v", srcPath, err) + } else if actual != expected { + log.Printf("integrity check FAILED for upload %s: expected %s, got %s", info.ID, expected, actual) + ep.queries.FailUpload(context.Background(), info.ID) + os.Remove(srcPath) + return "", false + } else { + log.Printf("integrity verified for %s (SHA-256: %s)", info.ID, actual) + } + } + + // The row keeps the sanitized name so downloads recompute this exact key — + // see objectstore.Client.Key. + key := ep.store.Key(info.ID, filename) + if err := ep.store.Upload(context.Background(), key, srcPath); err != nil { + // Leave the temp file: the bytes are intact, so a retry has something to + // work with. + log.Printf("error uploading %s to S3: %v", info.ID, err) + ep.queries.FailUpload(context.Background(), info.ID) + return "", false + } + + if err := os.Remove(srcPath); err != nil { + log.Printf("warning: failed to remove temp file %s after S3 upload: %v", srcPath, err) + } + log.Printf("upload saved: s3://%s/%s", ep.store.Bucket(), key) + return filename, true +} + +func (ep *EventProcessor) storeToDisk(info handler.FileInfo, completedAt time.Time) (string, bool) { // Resolve the target row from the DB — targets can be added/edited by admins // at runtime, so this can't be cached at startup. When the target is bound to // a hardcoded form, the final filename is derived from the submitted form @@ -187,17 +509,9 @@ func (ep *EventProcessor) finalizeUpload(info handler.FileInfo, completedAt time dstPath = ep.renameUpload(info.ID, filename, targetDir) } } - - // Record the final on-disk name (form-derived and/or de-duped) so the upload - // history reflects what actually landed in the target dir, not the original - // client filename captured at create time. - if dstPath != "" { - if err := ep.queries.UpdateUploadFilename(context.Background(), db.UpdateUploadFilenameParams{ - Filename: filepath.Base(dstPath), - ID: info.ID, - }); err != nil { - log.Printf("warning: failed to update stored filename for %s: %v", info.ID, err) - } + if dstPath == "" { + ep.queries.FailUpload(context.Background(), info.ID) + return "", false } // Verify file integrity against the client-provided SHA-256 hash @@ -245,41 +559,10 @@ func (ep *EventProcessor) finalizeUpload(info handler.FileInfo, completedAt time } } - // For concatenated uploads, fix the duration to measure from the earliest - // partial upload's creation time (the final upload is created and completed - // in the same request, so its created_at == completed_at). - if info.PartialUploads != nil { - var earliest time.Time - for _, partialID := range info.PartialUploads { - p, err := ep.queries.GetUpload(context.Background(), partialID) - if err != nil { - continue - } - if earliest.IsZero() || p.CreatedAt.Before(earliest) { - earliest = p.CreatedAt - } - } - if !earliest.IsZero() { - durationMs := completedAt.Sub(earliest).Milliseconds() - ep.queries.UpdateDurationMs(context.Background(), db.UpdateDurationMsParams{ - DurationMs: sql.NullInt64{Int64: durationMs, Valid: true}, - ID: info.ID, - }) - } - - // Clean up partial files and .info files - for _, partialID := range info.PartialUploads { - os.Remove(filepath.Join(ep.tempDir, partialID)) - os.Remove(filepath.Join(ep.tempDir, partialID+".info")) - } - // Delete partial DB records - for _, partialID := range info.PartialUploads { - ep.queries.DeleteUpload(context.Background(), partialID) - } + if integrityFailed { + return "", false } - - // Remove the .info file for the completed upload - os.Remove(filepath.Join(ep.tempDir, info.ID+".info")) + return filepath.Base(dstPath), true } func (ep *EventProcessor) handleTerminated(event handler.HookEvent) { @@ -290,8 +573,12 @@ func (ep *EventProcessor) handleTerminated(event handler.HookEvent) { } } -// renameUpload moves the uploaded file from its hash-based ID to the original filename -// inside targetDir. If a file with the same name exists, a numeric suffix is added. +// renameUpload moves the uploaded file from its hash-based ID to the original +// filename inside targetDir. Destination selection and publication are one +// atomic, no-replace operation: two processes finishing the same filename can +// never both choose it and silently replace one another. If a name is already +// occupied, a numeric suffix is tried. +// // Returns the destination path on success, or empty string on failure. func (ep *EventProcessor) renameUpload(id, filename, targetDir string) string { if err := os.MkdirAll(targetDir, 0755); err != nil { @@ -300,7 +587,6 @@ func (ep *EventProcessor) renameUpload(id, filename, targetDir string) string { } src := filepath.Join(ep.tempDir, id) - dst := ep.uniquePath(targetDir, filename) // Defense in depth: verify the resolved destination is inside targetDir. // SanitizeFilename should already guarantee this, but a containment check @@ -312,9 +598,9 @@ func (ep *EventProcessor) renameUpload(id, filename, targetDir string) string { log.Printf("error resolving target dir %s: %v", targetDir, err) return "" } - absDst, err := filepath.Abs(dst) + absDst, err := filepath.Abs(filepath.Join(targetDir, filename)) if err != nil { - log.Printf("error resolving destination %s: %v", dst, err) + log.Printf("error resolving destination %s: %v", filename, err) return "" } rel, err := filepath.Rel(absTarget, absDst) @@ -323,63 +609,92 @@ func (ep *EventProcessor) renameUpload(id, filename, targetDir string) string { return "" } - if err := os.Rename(src, dst); err != nil { - if !errors.Is(err, syscall.EXDEV) { - log.Printf("error renaming upload %s to %s: %v", id, dst, err) - return "" - } - log.Printf("cross-device copy: %s -> %s", src, dst) - if err := crossDeviceMove(src, dst); err != nil { - log.Printf("error moving upload %s to %s across filesystems: %v", id, dst, err) - return "" - } + dst, err := moveUploadNoReplace(src, targetDir, filename) + if errors.Is(err, syscall.EXDEV) { + log.Printf("cross-device copy: %s -> %s", src, targetDir) + dst, err = crossDeviceMove(src, targetDir, filename) + } + if err != nil { + log.Printf("error moving upload %s into %s: %v", id, targetDir, err) + return "" } log.Printf("upload saved: %s", dst) return dst } -// crossDeviceMove copies src to dst when os.Rename fails with EXDEV. To keep -// the destination atomically visible, bytes are written to a sibling -// ".part" file (same filesystem as dst, so the closing rename is atomic), -// then renamed into place. The source is removed only after the destination is -// safely published; on any error the partial sibling is cleaned up and src is -// left in place so the upload can be retried. -func crossDeviceMove(src, dst string) error { +// moveUploadNoReplace atomically moves src to the first unoccupied destination +// name. renameNoReplace is implemented with the host OS's exclusive-rename +// primitive, so checking a candidate and claiming it cannot race with another +// process. +func moveUploadNoReplace(src, targetDir, filename string) (string, error) { + for suffix := 0; ; suffix++ { + dst := uploadCollisionPath(targetDir, filename, suffix) + err := renameNoReplace(src, dst) + if err == nil { + return dst, nil + } + if errors.Is(err, os.ErrExist) { + continue + } + return "", err + } +} + +// crossDeviceMove copies src into a private staging file on the destination +// filesystem, fsyncs it, and then uses the same atomic no-replace publication +// as the same-device path. The source is removed only after publication. On an +// error, the staging file is removed and src remains available for recovery. +func crossDeviceMove(src, targetDir, filename string) (string, error) { in, err := os.Open(src) if err != nil { - return fmt.Errorf("open source: %w", err) + return "", fmt.Errorf("open source: %w", err) } defer in.Close() - part := dst + ".part" - out, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) + out, err := os.CreateTemp(targetDir, ".filebox-upload-*.part") if err != nil { - return fmt.Errorf("create partial: %w", err) + return "", fmt.Errorf("create staging file: %w", err) } + part := out.Name() cleanup := func() { _ = os.Remove(part) } + if err := out.Chmod(0644); err != nil { + out.Close() + cleanup() + return "", fmt.Errorf("set staging permissions: %w", err) + } if _, err := io.Copy(out, in); err != nil { out.Close() cleanup() - return fmt.Errorf("copy: %w", err) + return "", fmt.Errorf("copy: %w", err) } if err := out.Sync(); err != nil { out.Close() cleanup() - return fmt.Errorf("sync: %w", err) + return "", fmt.Errorf("sync: %w", err) } if err := out.Close(); err != nil { cleanup() - return fmt.Errorf("close partial: %w", err) + return "", fmt.Errorf("close staging file: %w", err) } - if err := os.Rename(part, dst); err != nil { + dst, err := moveUploadNoReplace(part, targetDir, filename) + if err != nil { cleanup() - return fmt.Errorf("rename partial: %w", err) + return "", fmt.Errorf("publish staging file: %w", err) } if err := os.Remove(src); err != nil { log.Printf("warning: failed to remove source %s after cross-device move: %v", src, err) } - return nil + return dst, nil +} + +func uploadCollisionPath(dir, filename string, suffix int) string { + if suffix == 0 { + return filepath.Join(dir, filename) + } + ext := filepath.Ext(filename) + base := strings.TrimSuffix(filename, ext) + return filepath.Join(dir, fmt.Sprintf("%s (%d)%s", base, suffix, ext)) } // sidecarPayload is the JSON written next to a form upload, recording the @@ -482,22 +797,6 @@ func (ep *EventProcessor) fireWebhook(url, sidecarName, relPath string) { }() } -func (ep *EventProcessor) uniquePath(dir, filename string) string { - dst := filepath.Join(dir, filename) - if _, err := os.Stat(dst); os.IsNotExist(err) { - return dst - } - - ext := filepath.Ext(filename) - base := strings.TrimSuffix(filename, ext) - for i := 1; ; i++ { - dst = filepath.Join(dir, fmt.Sprintf("%s (%d)%s", base, i, ext)) - if _, err := os.Stat(dst); os.IsNotExist(err) { - return dst - } - } -} - // SanitizeFilename returns a filename containing only [A-Za-z0-9_-] plus an // optional single '.' separating the extension. Every other rune (including // internal dots) is replaced with '_'. A leading '.' is never treated as an diff --git a/internal/tus/hooks_test.go b/internal/tus/hooks_test.go index cb33d14..3300108 100644 --- a/internal/tus/hooks_test.go +++ b/internal/tus/hooks_test.go @@ -1,6 +1,12 @@ package tus -import "testing" +import ( + "fmt" + "os" + "path/filepath" + "sync" + "testing" +) func TestSanitizeFilename(t *testing.T) { cases := []struct { @@ -41,3 +47,110 @@ func TestSanitizeFilename(t *testing.T) { } } } + +func TestRenameUploadConcurrentSameNameNeverReplaces(t *testing.T) { + tempDir := t.TempDir() + targetDir := t.TempDir() + ep := &EventProcessor{tempDir: tempDir} + + const uploads = 48 + type result struct { + path string + payload string + } + results := make(chan result, uploads) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < uploads; i++ { + id := fmt.Sprintf("upload-%02d", i) + payload := fmt.Sprintf("unique payload %02d", i) + if err := os.WriteFile(filepath.Join(tempDir, id), []byte(payload), 0644); err != nil { + t.Fatalf("write source %s: %v", id, err) + } + + wg.Add(1) + go func() { + defer wg.Done() + <-start + results <- result{path: ep.renameUpload(id, "same-name.bin", targetDir), payload: payload} + }() + } + close(start) + wg.Wait() + close(results) + + seen := make(map[string]string, uploads) + for got := range results { + if got.path == "" { + t.Fatal("concurrent rename failed") + } + if previous, duplicate := seen[got.path]; duplicate { + t.Fatalf("destination %q was claimed twice for %q and %q", got.path, previous, got.payload) + } + seen[got.path] = got.payload + contents, err := os.ReadFile(got.path) + if err != nil { + t.Fatalf("read destination %q: %v", got.path, err) + } + if string(contents) != got.payload { + t.Fatalf("destination %q contains %q, want %q", got.path, contents, got.payload) + } + } + if len(seen) != uploads { + t.Fatalf("published %d unique files, want %d", len(seen), uploads) + } +} + +func TestCrossDeviceMoveConcurrentSameNameNeverReplaces(t *testing.T) { + sourceDir := t.TempDir() + targetDir := t.TempDir() + + const uploads = 24 + type result struct { + path string + payload string + err error + } + results := make(chan result, uploads) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < uploads; i++ { + src := filepath.Join(sourceDir, fmt.Sprintf("upload-%02d", i)) + payload := fmt.Sprintf("cross-device payload %02d", i) + if err := os.WriteFile(src, []byte(payload), 0644); err != nil { + t.Fatalf("write source %q: %v", src, err) + } + + wg.Add(1) + go func() { + defer wg.Done() + <-start + path, err := crossDeviceMove(src, targetDir, "same-name.bin") + results <- result{path: path, payload: payload, err: err} + }() + } + close(start) + wg.Wait() + close(results) + + seen := make(map[string]string, uploads) + for got := range results { + if got.err != nil { + t.Fatalf("cross-device publication failed: %v", got.err) + } + if previous, duplicate := seen[got.path]; duplicate { + t.Fatalf("destination %q was claimed twice for %q and %q", got.path, previous, got.payload) + } + seen[got.path] = got.payload + contents, err := os.ReadFile(got.path) + if err != nil { + t.Fatalf("read destination %q: %v", got.path, err) + } + if string(contents) != got.payload { + t.Fatalf("destination %q contains %q, want %q", got.path, contents, got.payload) + } + } + if len(seen) != uploads { + t.Fatalf("published %d unique files, want %d", len(seen), uploads) + } +} diff --git a/internal/tus/rename_noreplace_darwin.go b/internal/tus/rename_noreplace_darwin.go new file mode 100644 index 0000000..2123def --- /dev/null +++ b/internal/tus/rename_noreplace_darwin.go @@ -0,0 +1,11 @@ +//go:build darwin + +package tus + +import "golang.org/x/sys/unix" + +// renameNoReplace is Darwin's atomic exclusive rename. A filesystem that does +// not implement RENAME_EXCL is rejected rather than risking an overwrite. +func renameNoReplace(oldPath, newPath string) error { + return unix.RenamexNp(oldPath, newPath, unix.RENAME_EXCL) +} diff --git a/internal/tus/rename_noreplace_fallback.go b/internal/tus/rename_noreplace_fallback.go new file mode 100644 index 0000000..41cfb69 --- /dev/null +++ b/internal/tus/rename_noreplace_fallback.go @@ -0,0 +1,22 @@ +//go:build !linux && !darwin + +package tus + +import "os" + +// renameNoReplace falls back to an atomic no-overwrite hard-link claim on +// platforms without a native exclusive rename. This requires oldPath and +// newPath to be on a filesystem that supports hard links. The Linux and Darwin +// builds use native atomic rename operations and do not have this limitation. +func renameNoReplace(oldPath, newPath string) error { + if err := os.Link(oldPath, newPath); err != nil { + return err + } + if err := os.Remove(oldPath); err != nil { + // Preserve move semantics on a reported failure. Removing the link cannot + // lose data because oldPath still names the same inode. + _ = os.Remove(newPath) + return err + } + return nil +} diff --git a/internal/tus/rename_noreplace_linux.go b/internal/tus/rename_noreplace_linux.go new file mode 100644 index 0000000..f24171a --- /dev/null +++ b/internal/tus/rename_noreplace_linux.go @@ -0,0 +1,12 @@ +//go:build linux + +package tus + +import "golang.org/x/sys/unix" + +// renameNoReplace atomically moves oldPath to newPath only when newPath does +// not exist. Linux has provided renameat2(RENAME_NOREPLACE) since 3.15; failing +// closed on an older kernel/filesystem is safer than silently replacing data. +func renameNoReplace(oldPath, newPath string) error { + return unix.Renameat2(unix.AT_FDCWD, oldPath, unix.AT_FDCWD, newPath, unix.RENAME_NOREPLACE) +}