1 files changed, 534 insertions, 0 deletions
diff --git a/fs/operations/copy_test.go b/fs/operations/copy_test.go
new file mode 100644
index 0000000..e717f3b
--- /dev/null
+++ b/fs/operations/copy_test.go
@@ -0,0 +1,534 @@
+package operations_test
+
+import (
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"runtime"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fs/sync"
+	"github.com/rclone/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTruncateString(t *testing.T) {
+	for _, test := range []struct {
+		in   string
+		n    int
+		want string
+	}{
+		{
+			in:   "",
+			n:    0,
+			want: "",
+		}, {
+			in:   "Hello World",
+			n:    5,
+			want: "Hello",
+		}, {
+			in:   "ááááá",
+			n:    5,
+			want: "áá",
+		}, {
+			in:   "ááááá\xFF\xFF",
+			n:    5,
+			want: "áá\xc3",
+		}, {
+			in:   "世世世世世",
+			n:    7,
+			want: "世世",
+		}, {
+			in:   "🙂🙂🙂🙂🙂",
+			n:    16,
+			want: "🙂🙂🙂🙂",
+		}, {
+			in:   "🙂🙂🙂🙂🙂",
+			n:    15,
+			want: "🙂🙂🙂",
+		}, {
+			in:   "🙂🙂🙂🙂🙂",
+			n:    14,
+			want: "🙂🙂🙂",
+		}, {
+			in:   "🙂🙂🙂🙂🙂",
+			n:    13,
+			want: "🙂🙂🙂",
+		}, {
+			in:   "🙂🙂🙂🙂🙂",
+			n:    12,
+			want: "🙂🙂🙂",
+		}, {
+			in:   "🙂🙂🙂🙂🙂",
+			n:    11,
+			want: "🙂🙂",
+		}, {
+			in:   "𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
+			n:    100,
+			want: "𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢ",
+		}, {
+			in:   "a𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
+			n:    100,
+			want: "a𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢ",
+		}, {
+			in:   "aa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
+			n:    100,
+			want: "aa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱ",
+		}, {
+			in:   "aaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
+			n:    100,
+			want: "aaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱ",
+		}, {
+			in:   "aaaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
+			n:    100,
+			want: "aaaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽ",
+		},
+	} {
+		got := operations.TruncateString(test.in, test.n)
+		assert.Equal(t, test.want, got, fmt.Sprintf("In %q", test.in))
+		assert.LessOrEqual(t, len(got), test.n)
+		assert.GreaterOrEqual(t, len(got), test.n-3)
+	}
+}
+
+func TestCopyFile(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+
+	file1 := r.WriteFile("file1", "file1 contents", t1)
+	r.CheckLocalItems(t, file1)
+
+	file2 := file1
+	file2.Path = "sub/file2"
+
+	err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, file1)
+	r.CheckRemoteItems(t, file2)
+
+	err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, file1)
+	r.CheckRemoteItems(t, file2)
+
+	err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, file1)
+	r.CheckRemoteItems(t, file2)
+}
+
+// Find the longest file name for writing to local
+func maxLengthFileName(t *testing.T, r *fstest.Run) string {
+	require.NoError(t, r.Flocal.Mkdir(context.Background(), "")) // create the root
+	const maxLen = 16 * 1024
+	name := strings.Repeat("A", maxLen)
+	i := sort.Search(len(name), func(i int) (fail bool) {
+		filePath := path.Join(r.LocalName, name[:i])
+		err := os.WriteFile(filePath, []byte{0}, 0777)
+		if err != nil {
+			return true
+		}
+		err = os.Remove(filePath)
+		if err != nil {
+			t.Logf("Failed to remove test file: %v", err)
+		}
+		return false
+	})
+	return name[:i-1]
+}
+
+// Check we can copy a file of maximum name length
+func TestCopyLongFile(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	if !r.Fremote.Features().IsLocal {
+		t.Skip("Test only runs on local")
+	}
+
+	// Find the maximum length of file we can write
+	name := maxLengthFileName(t, r)
+	t.Logf("Max length of file name is %d", len(name))
+	file1 := r.WriteFile(name, "file1 contents", t1)
+	r.CheckLocalItems(t, file1)
+
+	err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, file1)
+	r.CheckRemoteItems(t, file1)
+}
+
+func TestCopyFileBackupDir(t *testing.T) {
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	r := fstest.NewRun(t)
+	if !operations.CanServerSideMove(r.Fremote) {
+		t.Skip("Skipping test as remote does not support server-side move or copy")
+	}
+
+	ci.BackupDir = r.FremoteName + "/backup"
+
+	file1 := r.WriteFile("dst/file1", "file1 contents", t1)
+	r.CheckLocalItems(t, file1)
+
+	file1old := r.WriteObject(ctx, "dst/file1", "file1 contents old", t1)
+	r.CheckRemoteItems(t, file1old)
+
+	err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, file1)
+	file1old.Path = "backup/dst/file1"
+	r.CheckRemoteItems(t, file1old, file1)
+}
+
+// Test with CompareDest set
+func TestCopyFileCompareDest(t *testing.T) {
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	r := fstest.NewRun(t)
+
+	ci.CompareDest = []string{r.FremoteName + "/CompareDest"}
+	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
+	require.NoError(t, err)
+
+	// check empty dest, empty compare
+	file1 := r.WriteFile("one", "one", t1)
+	r.CheckLocalItems(t, file1)
+
+	err = operations.CopyFile(ctx, fdst, r.Flocal, file1.Path, file1.Path)
+	require.NoError(t, err)
+
+	file1dst := file1
+	file1dst.Path = "dst/one"
+
+	r.CheckRemoteItems(t, file1dst)
+
+	// check old dest, empty compare
+	file1b := r.WriteFile("one", "onet2", t2)
+	r.CheckRemoteItems(t, file1dst)
+	r.CheckLocalItems(t, file1b)
+
+	err = operations.CopyFile(ctx, fdst, r.Flocal, file1b.Path, file1b.Path)
+	require.NoError(t, err)
+
+	file1bdst := file1b
+	file1bdst.Path = "dst/one"
+
+	r.CheckRemoteItems(t, file1bdst)
+
+	// check old dest, new compare
+	file3 := r.WriteObject(ctx, "dst/one", "one", t1)
+	file2 := r.WriteObject(ctx, "CompareDest/one", "onet2", t2)
+	file1c := r.WriteFile("one", "onet2", t2)
+	r.CheckRemoteItems(t, file2, file3)
+	r.CheckLocalItems(t, file1c)
+
+	err = operations.CopyFile(ctx, fdst, r.Flocal, file1c.Path, file1c.Path)
+	require.NoError(t, err)
+
+	r.CheckRemoteItems(t, file2, file3)
+
+	// check empty dest, new compare
+	file4 := r.WriteObject(ctx, "CompareDest/two", "two", t2)
+	file5 := r.WriteFile("two", "two", t2)
+	r.CheckRemoteItems(t, file2, file3, file4)
+	r.CheckLocalItems(t, file1c, file5)
+
+	err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
+	require.NoError(t, err)
+
+	r.CheckRemoteItems(t, file2, file3, file4)
+
+	// check new dest, new compare
+	err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
+	require.NoError(t, err)
+
+	r.CheckRemoteItems(t, file2, file3, file4)
+
+	// check empty dest, old compare
+	file5b := r.WriteFile("two", "twot3", t3)
+	r.CheckRemoteItems(t, file2, file3, file4)
+	r.CheckLocalItems(t, file1c, file5b)
+
+	err = operations.CopyFile(ctx, fdst, r.Flocal, file5b.Path, file5b.Path)
+	require.NoError(t, err)
+
+	file5bdst := file5b
+	file5bdst.Path = "dst/two"
+
+	r.CheckRemoteItems(t, file2, file3, file4, file5bdst)
+}
+
+// Test with CopyDest set
+func TestCopyFileCopyDest(t *testing.T) {
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	r := fstest.NewRun(t)
+
+	if r.Fremote.Features().Copy == nil {
+		t.Skip("Skipping test as remote does not support server-side copy")
+	}
+
+	ci.CopyDest = []string{r.FremoteName + "/CopyDest"}
+
+	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
+	require.NoError(t, err)
+
+	// check empty dest, empty copy
+	file1 := r.WriteFile("one", "one", t1)
+	r.CheckLocalItems(t, file1)
+
+	err = operations.CopyFile(ctx, fdst, r.Flocal, file1.Path, file1.Path)
+	require.NoError(t, err)
+
+	file1dst := file1
+	file1dst.Path = "dst/one"
+
+	r.CheckRemoteItems(t, file1dst)
+
+	// check old dest, empty copy
+	file1b := r.WriteFile("one", "onet2", t2)
+	r.CheckRemoteItems(t, file1dst)
+	r.CheckLocalItems(t, file1b)
+
+	err = operations.CopyFile(ctx, fdst, r.Flocal, file1b.Path, file1b.Path)
+	require.NoError(t, err)
+
+	file1bdst := file1b
+	file1bdst.Path = "dst/one"
+
+	r.CheckRemoteItems(t, file1bdst)
+
+	// check old dest, new copy, backup-dir
+
+	ci.BackupDir = r.FremoteName + "/BackupDir"
+
+	file3 := r.WriteObject(ctx, "dst/one", "one", t1)
+	file2 := r.WriteObject(ctx, "CopyDest/one", "onet2", t2)
+	file1c := r.WriteFile("one", "onet2", t2)
+	r.CheckRemoteItems(t, file2, file3)
+	r.CheckLocalItems(t, file1c)
+
+	err = operations.CopyFile(ctx, fdst, r.Flocal, file1c.Path, file1c.Path)
+	require.NoError(t, err)
+
+	file2dst := file2
+	file2dst.Path = "dst/one"
+	file3.Path = "BackupDir/one"
+
+	r.CheckRemoteItems(t, file2, file2dst, file3)
+	ci.BackupDir = ""
+
+	// check empty dest, new copy
+	file4 := r.WriteObject(ctx, "CopyDest/two", "two", t2)
+	file5 := r.WriteFile("two", "two", t2)
+	r.CheckRemoteItems(t, file2, file2dst, file3, file4)
+	r.CheckLocalItems(t, file1c, file5)
+
+	err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
+	require.NoError(t, err)
+
+	file4dst := file4
+	file4dst.Path = "dst/two"
+
+	r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst)
+
+	// check new dest, new copy
+	err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
+	require.NoError(t, err)
+
+	r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst)
+
+	// check empty dest, old copy
+	file6 := r.WriteObject(ctx, "CopyDest/three", "three", t2)
+	file7 := r.WriteFile("three", "threet3", t3)
+	r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6)
+	r.CheckLocalItems(t, file1c, file5, file7)
+
+	err = operations.CopyFile(ctx, fdst, r.Flocal, file7.Path, file7.Path)
+	require.NoError(t, err)
+
+	file7dst := file7
+	file7dst.Path = "dst/three"
+
+	r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6, file7dst)
+}
+
+func TestCopyInplace(t *testing.T) {
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	r := fstest.NewRun(t)
+
+	if !r.Fremote.Features().PartialUploads {
+		t.Skip("Partial uploads not supported")
+	}
+
+	ci.Inplace = true
+
+	file1 := r.WriteFile("file1", "file1 contents", t1)
+	r.CheckLocalItems(t, file1)
+
+	file2 := file1
+	file2.Path = "sub/file2"
+
+	err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, file1)
+	r.CheckRemoteItems(t, file2)
+
+	err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, file1)
+	r.CheckRemoteItems(t, file2)
+
+	err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, file1)
+	r.CheckRemoteItems(t, file2)
+}
+
+func TestCopyLongFileName(t *testing.T) {
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	r := fstest.NewRun(t)
+
+	if !r.Fremote.Features().PartialUploads {
+		t.Skip("Partial uploads not supported")
+	}
+
+	ci.Inplace = false // the default
+
+	file1 := r.WriteFile("file1", "file1 contents", t1)
+	r.CheckLocalItems(t, file1)
+
+	file2 := file1
+	file2.Path = "sub/" + strings.Repeat("file2", 30)
+
+	err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, file1)
+	r.CheckRemoteItems(t, file2)
+
+	err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, file1)
+	r.CheckRemoteItems(t, file2)
+
+	err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, file1)
+	r.CheckRemoteItems(t, file2)
+}
+
+func TestCopyLongFileNameCollision(t *testing.T) {
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	r := fstest.NewRun(t)
+
+	if !r.Fremote.Features().PartialUploads {
+		t.Skip("Partial uploads not supported")
+	}
+
+	ci.Inplace = false
+	ci.Transfers = 4
+
+	// Write a lot of identical files with long names
+	files := make([]fstest.Item, 10)
+	namePrefix := strings.Repeat("file1", 30)
+	for i := range files {
+		files[i] = r.WriteFile(fmt.Sprintf("%s%02d", namePrefix, i), "file1 contents", t1)
+	}
+	r.CheckLocalItems(t, files...)
+
+	err := sync.CopyDir(ctx, r.Fremote, r.Flocal, false)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, files...)
+	r.CheckRemoteItems(t, files...)
+}
+
+func TestCopyFileMaxTransfer(t *testing.T) {
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	r := fstest.NewRun(t)
+	defer accounting.Stats(ctx).ResetCounters()
+
+	const sizeCutoff = 2048
+
+	// Make random incompressible data
+	randomData := make([]byte, sizeCutoff)
+	_, err := rand.Read(randomData)
+	require.NoError(t, err)
+	randomString := string(randomData)
+
+	file1 := r.WriteFile("TestCopyFileMaxTransfer/file1", "file1 contents", t1)
+	file2 := r.WriteFile("TestCopyFileMaxTransfer/file2", "file2 contents"+randomString, t2)
+	file3 := r.WriteFile("TestCopyFileMaxTransfer/file3", "file3 contents"+randomString, t2)
+	file4 := r.WriteFile("TestCopyFileMaxTransfer/file4", "file4 contents"+randomString, t2)
+
+	// Cutoff mode: Hard
+	ci.MaxTransfer = sizeCutoff
+	ci.CutoffMode = fs.CutoffModeHard
+
+	if runtime.GOOS == "darwin" {
+		// disable server-side copies as they don't count towards transfer size stats
+		r.Flocal.Features().Disable("Copy")
+		if r.Fremote.Features().IsLocal {
+			r.Fremote.Features().Disable("Copy")
+		}
+	}
+
+	// file1: Show a small file gets transferred OK
+	accounting.Stats(ctx).ResetCounters()
+	err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, file1, file2, file3, file4)
+	r.CheckRemoteItems(t, file1)
+
+	// file2: show a large file does not get transferred
+	accounting.Stats(ctx).ResetCounters()
+	err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file2.Path)
+	require.NotNil(t, err, "Did not get expected max transfer limit error")
+	if !errors.Is(err, accounting.ErrorMaxTransferLimitReachedFatal) {
+		t.Log("Expecting error to contain accounting.ErrorMaxTransferLimitReachedFatal")
+		// Sometimes the backends or their SDKs don't pass the
+		// error through properly, so check that it at least
+		// has the text we expect in.
+		assert.Contains(t, err.Error(), "max transfer limit reached")
+	}
+	r.CheckLocalItems(t, file1, file2, file3, file4)
+	r.CheckRemoteItems(t, file1)
+
+	// Cutoff mode: Cautious
+	ci.CutoffMode = fs.CutoffModeCautious
+
+	// file3: show a large file does not get transferred
+	accounting.Stats(ctx).ResetCounters()
+	err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file3.Path, file3.Path)
+	require.NotNil(t, err)
+	assert.True(t, errors.Is(err, accounting.ErrorMaxTransferLimitReachedGraceful))
+	r.CheckLocalItems(t, file1, file2, file3, file4)
+	r.CheckRemoteItems(t, file1)
+
+	if isChunker(r.Fremote) {
+		t.Log("skipping remainder of test for chunker as it involves multiple transfers")
+		return
+	}
+
+	// Cutoff mode: Soft
+	ci.CutoffMode = fs.CutoffModeSoft
+
+	// file4: show a large file does get transferred this time
+	accounting.Stats(ctx).ResetCounters()
+	err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file4.Path, file4.Path)
+	require.NoError(t, err)
+	r.CheckLocalItems(t, file1, file2, file3, file4)
+	r.CheckRemoteItems(t, file1, file4)
+}