From d8745b3fdaf473cf2dfd88ec96f72d055a74da08 Mon Sep 17 00:00:00 2001 From: Andrey Markelov Date: Fri, 26 Jun 2026 12:55:13 -0700 Subject: [PATCH] Add upper-bound chunk-size validation to put command Reject chunk sizes below 4MiB, not a multiple of 4MiB, or above 128MiB with distinct error messages. Previously only the modulo check existed, silently allowing values that exceed Dropbox upload-session limits. Update flag descriptions, examples, and generated docs to document the constraints. --- README.md | 11 ++++++++++- cmd/put.go | 23 +++++++++++++++++++---- cmd/put_test.go | 37 +++++++++++++++++++++++++++++-------- docs/commands/dbxcli_put.md | 8 ++++++-- 4 files changed, 64 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index c39501b..d513060 100644 --- a/README.md +++ b/README.md @@ -406,12 +406,21 @@ The `--verbose` option will turn on verbose logging and is useful for debugging. ```sh $ dbxcli put file.txt /destination/file.txt # upload a single file $ dbxcli put -r ./project /backup/project # recursively upload a directory -$ dbxcli put -r -w 8 ./large-folder /backup/large # use 8 workers per large file +$ dbxcli put -w 1 -c 134217728 large.zip /backup/large.zip $ dbxcli put --if-exists skip file.txt /dest.txt # skip if the file already exists ``` By default, `put` overwrites existing destination files. Use `--if-exists overwrite|skip|fail` to choose whether existing files are overwritten, skipped, or treated as an error. +For files larger than 32MiB, `put` uses Dropbox upload sessions. By default, it +uses 4 workers and 16MiB chunks. Each chunk is one upload-session request; chunk +size must be a multiple of 4MiB and no more than 128MiB. On unstable networks, +fewer workers and larger chunks may be more reliable: + +```sh +$ dbxcli put -w 1 -c 134217728 large.zip /backup/large.zip +``` + ### Downloading files and directories ```sh diff --git a/cmd/put.go b/cmd/put.go index b52fc46..e722b97 100644 --- a/cmd/put.go +++ b/cmd/put.go @@ -38,6 +38,11 @@ import ( const singleShotUploadSizeCutoff int64 = 32 * (1 << 20) const ( + putChunkSizeUnit int64 = 1 << 22 + // Dropbox upload-session requests should stay below 150 MB. Use a + // conservative 128 MiB max that is also a multiple of 4 MiB. + putMaxChunkSize int64 = 128 * (1 << 20) + putIfExistsOverwrite = "overwrite" putIfExistsSkip = "skip" putIfExistsFail = "fail" @@ -445,8 +450,14 @@ func parsePutOptions(cmd *cobra.Command) (putOptions, error) { if err != nil { return putOptions{}, err } - if chunkSize%(1<<22) != 0 { - return putOptions{}, invalidArgumentsErrorWithDetails("`put` requires chunk size to be multiple of 4MiB", flagErrorDetails("chunksize")) + if chunkSize < putChunkSizeUnit { + return putOptions{}, invalidArgumentsErrorWithDetails("`put` requires chunk size to be at least 4MiB", flagErrorDetails("chunksize")) + } + if chunkSize%putChunkSizeUnit != 0 { + return putOptions{}, invalidArgumentsErrorWithDetails("`put` requires chunk size to be a multiple of 4MiB", flagErrorDetails("chunksize")) + } + if chunkSize > putMaxChunkSize { + return putOptions{}, invalidArgumentsErrorWithDetails("`put` requires chunk size to be no more than 128MiB", flagErrorDetails("chunksize")) } workers, err := cmd.Flags().GetInt("workers") if err != nil { @@ -889,9 +900,13 @@ var putCmd = &cobra.Command{ - Use - as source to read from stdin (target is required). Stdin is spooled to a temp file before upload and may use disk space up to the full input size. + - Files larger than 32MiB use Dropbox upload sessions. Each chunk is one + upload-session request; chunk size must be a multiple of 4MiB and no more + than 128MiB. `, Example: ` dbxcli put file.txt /destination/file.txt dbxcli put -r ./project /backup/project + dbxcli put -w 1 -c 134217728 large.zip /backup/large.zip printf 'hello' | dbxcli put - /hello.txt tar cz ./src | dbxcli put - /backups/src.tgz`, RunE: put, @@ -901,8 +916,8 @@ func init() { RootCmd.AddCommand(putCmd) enableStructuredOutput(putCmd) putCmd.Flags().BoolP("recursive", "r", false, "Recursively upload directories") - putCmd.Flags().IntP("workers", "w", 4, "Number of concurrent upload workers to use") - putCmd.Flags().Int64P("chunksize", "c", 1<<24, "Chunk size to use (should be multiple of 4MiB)") + putCmd.Flags().IntP("workers", "w", 4, "Number of concurrent upload workers for chunked large-file uploads") + putCmd.Flags().Int64P("chunksize", "c", 1<<24, "Chunk size in bytes for chunked large-file uploads; must be a multiple of 4MiB and no more than 128MiB") putCmd.Flags().BoolP("debug", "d", false, "Print debug timing") putCmd.Flags().String("if-exists", putIfExistsOverwrite, "What to do when the destination file exists: overwrite, skip, or fail") } diff --git a/cmd/put_test.go b/cmd/put_test.go index fc506a7..3049e35 100644 --- a/cmd/put_test.go +++ b/cmd/put_test.go @@ -667,16 +667,37 @@ func TestPutRecursive_SkipsSymlinks(t *testing.T) { } func TestPutChunkSizeValidation(t *testing.T) { - tmpFile := filepath.Join(t.TempDir(), "test.txt") - if err := os.WriteFile(tmpFile, []byte("test"), 0644); err != nil { - t.Fatal(err) + tests := []struct { + name string + chunkSize string + want string + }{ + { + name: "below 4MiB", + chunkSize: "100", + want: "`put` requires chunk size to be at least 4MiB", + }, + { + name: "not multiple of 4MiB", + chunkSize: "6291456", + want: "`put` requires chunk size to be a multiple of 4MiB", + }, + { + name: "above Dropbox request limit", + chunkSize: "268435456", + want: "`put` requires chunk size to be no more than 128MiB", + }, } - cmd := testPutCmd() - _ = cmd.Flags().Set("chunksize", "100") - err := put(cmd, []string{tmpFile, "/test.txt"}) - if err == nil || err.Error() != "`put` requires chunk size to be multiple of 4MiB" { - t.Errorf("expected chunk size validation error, got %v", err) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmd := testPutCmd() + _ = cmd.Flags().Set("chunksize", tt.chunkSize) + _, err := parsePutOptions(cmd) + if err == nil || err.Error() != tt.want { + t.Errorf("expected chunk size validation error %q, got %v", tt.want, err) + } + }) } } diff --git a/docs/commands/dbxcli_put.md b/docs/commands/dbxcli_put.md index 9248fc2..5df0abf 100644 --- a/docs/commands/dbxcli_put.md +++ b/docs/commands/dbxcli_put.md @@ -12,6 +12,9 @@ Upload files or directories to Dropbox. - Use - as source to read from stdin (target is required). Stdin is spooled to a temp file before upload and may use disk space up to the full input size. + - Files larger than 32MiB use Dropbox upload sessions. Each chunk is one + upload-session request; chunk size must be a multiple of 4MiB and no more + than 128MiB. ``` @@ -23,6 +26,7 @@ dbxcli put [flags] [] ``` dbxcli put file.txt /destination/file.txt dbxcli put -r ./project /backup/project + dbxcli put -w 1 -c 134217728 large.zip /backup/large.zip printf 'hello' | dbxcli put - /hello.txt tar cz ./src | dbxcli put - /backups/src.tgz ``` @@ -30,12 +34,12 @@ dbxcli put [flags] [] ### Options ``` - -c, --chunksize int Chunk size to use (should be multiple of 4MiB) (default 16777216) + -c, --chunksize int Chunk size in bytes for chunked large-file uploads; must be a multiple of 4MiB and no more than 128MiB (default 16777216) -d, --debug Print debug timing -h, --help help for put --if-exists string What to do when the destination file exists: overwrite, skip, or fail (default "overwrite") -r, --recursive Recursively upload directories - -w, --workers int Number of concurrent upload workers to use (default 4) + -w, --workers int Number of concurrent upload workers for chunked large-file uploads (default 4) ``` ### Options inherited from parent commands