aboutsummaryrefslogtreecommitdiff
path: root/fs/operations
diff options
context:
space:
mode:
Diffstat (limited to 'fs/operations')
-rw-r--r--fs/operations/check.go626
-rw-r--r--fs/operations/check_test.go617
-rw-r--r--fs/operations/copy.go422
-rw-r--r--fs/operations/copy_test.go534
-rw-r--r--fs/operations/dedupe.go506
-rw-r--r--fs/operations/dedupe_test.go280
-rw-r--r--fs/operations/listdirsorted_test.go126
-rw-r--r--fs/operations/logger.go384
-rw-r--r--fs/operations/lsjson.go348
-rw-r--r--fs/operations/lsjson_test.go406
-rw-r--r--fs/operations/multithread.go380
-rw-r--r--fs/operations/multithread_test.go333
-rw-r--r--fs/operations/operations.go2742
-rw-r--r--fs/operations/operations_internal_test.go44
-rw-r--r--fs/operations/operations_test.go1968
-rw-r--r--fs/operations/operationsflags/operationsflags.go147
-rw-r--r--fs/operations/operationsflags/operationsflags.md40
-rw-r--r--fs/operations/rc.go1016
-rw-r--r--fs/operations/rc_test.go892
-rw-r--r--fs/operations/reopen.go346
-rw-r--r--fs/operations/reopen_test.go429
21 files changed, 12586 insertions, 0 deletions
diff --git a/fs/operations/check.go b/fs/operations/check.go
new file mode 100644
index 0000000..d4d1eb3
--- /dev/null
+++ b/fs/operations/check.go
@@ -0,0 +1,626 @@
+package operations
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "regexp"
+ "strings"
+ "sync"
+ "sync/atomic"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/filter"
+ "github.com/rclone/rclone/fs/fserrors"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/march"
+ "github.com/rclone/rclone/lib/readers"
+ "golang.org/x/text/unicode/norm"
+)
+
+// checkFn is the type of the checking function used in CheckFn()
+//
+// It should check the two objects (a, b) and return if they differ
+// and whether the hash was used.
+//
+// If there are differences then this should Errorf the difference and
+// the reason but return with err = nil. It should not CountError in
+// this case.
+type checkFn func(ctx context.Context, a, b fs.Object) (differ bool, noHash bool, err error)
+
+// CheckOpt contains options for the Check functions
+type CheckOpt struct {
+ Fdst, Fsrc fs.Fs // fses to check
+ Check checkFn // function to use for checking
+ OneWay bool // one way only?
+ Combined io.Writer // a file with file names with leading sigils
+ MissingOnSrc io.Writer // files only in the destination
+ MissingOnDst io.Writer // files only in the source
+ Match io.Writer // matching files
+ Differ io.Writer // differing files
+ Error io.Writer // files with errors of some kind
+}
+
+// checkMarch is used to march over two Fses in the same way as
+// sync/copy
+type checkMarch struct {
+ ctx context.Context
+ ioMu sync.Mutex
+ wg sync.WaitGroup
+ tokens chan struct{}
+ differences atomic.Int32
+ noHashes atomic.Int32
+ srcFilesMissing atomic.Int32
+ dstFilesMissing atomic.Int32
+ matches atomic.Int32
+ opt CheckOpt
+}
+
+// report outputs the fileName to out if required and to the combined log
+func (c *checkMarch) report(o fs.DirEntry, out io.Writer, sigil rune) {
+ c.reportFilename(o.String(), out, sigil)
+}
+
+func (c *checkMarch) reportFilename(filename string, out io.Writer, sigil rune) {
+ if out != nil {
+ SyncFprintf(out, "%s\n", filename)
+ }
+ if c.opt.Combined != nil {
+ SyncFprintf(c.opt.Combined, "%c %s\n", sigil, filename)
+ }
+}
+
+// DstOnly have an object which is in the destination only
+func (c *checkMarch) DstOnly(dst fs.DirEntry) (recurse bool) {
+ switch dst.(type) {
+ case fs.Object:
+ if c.opt.OneWay {
+ return false
+ }
+ err := fmt.Errorf("file not in %v", c.opt.Fsrc)
+ fs.Errorf(dst, "%v", err)
+ _ = fs.CountError(c.ctx, err)
+ c.differences.Add(1)
+ c.srcFilesMissing.Add(1)
+ c.report(dst, c.opt.MissingOnSrc, '-')
+ case fs.Directory:
+ // Do the same thing to the entire contents of the directory
+ if c.opt.OneWay {
+ return false
+ }
+ return true
+ default:
+ panic("Bad object in DirEntries")
+ }
+ return false
+}
+
+// SrcOnly have an object which is in the source only
+func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
+ switch src.(type) {
+ case fs.Object:
+ err := fmt.Errorf("file not in %v", c.opt.Fdst)
+ fs.Errorf(src, "%v", err)
+ _ = fs.CountError(c.ctx, err)
+ c.differences.Add(1)
+ c.dstFilesMissing.Add(1)
+ c.report(src, c.opt.MissingOnDst, '+')
+ case fs.Directory:
+ // Do the same thing to the entire contents of the directory
+ return true
+ default:
+ panic("Bad object in DirEntries")
+ }
+ return false
+}
+
+// check to see if two objects are identical using the check function
+func (c *checkMarch) checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
+ ci := fs.GetConfig(ctx)
+ tr := accounting.Stats(ctx).NewCheckingTransfer(src, "checking")
+ defer func() {
+ tr.Done(ctx, err)
+ }()
+ if sizeDiffers(ctx, src, dst) {
+ err = fmt.Errorf("sizes differ")
+ fs.Errorf(src, "%v", err)
+ return true, false, nil
+ }
+ if ci.SizeOnly {
+ return false, false, nil
+ }
+ return c.opt.Check(ctx, dst, src)
+}
+
+// Match is called when src and dst are present, so sync src to dst
+func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse bool) {
+ switch srcX := src.(type) {
+ case fs.Object:
+ dstX, ok := dst.(fs.Object)
+ if ok {
+ if SkipDestructive(ctx, src, "check") {
+ return false
+ }
+ c.wg.Add(1)
+ c.tokens <- struct{}{} // put a token to limit concurrency
+ go func() {
+ defer func() {
+ <-c.tokens // get the token back to free up a slot
+ c.wg.Done()
+ }()
+ differ, noHash, err := c.checkIdentical(ctx, dstX, srcX)
+ if err != nil {
+ fs.Errorf(src, "%v", err)
+ _ = fs.CountError(ctx, err)
+ c.report(src, c.opt.Error, '!')
+ } else if differ {
+ c.differences.Add(1)
+ err := errors.New("files differ")
+ // the checkFn has already logged the reason
+ _ = fs.CountError(ctx, err)
+ c.report(src, c.opt.Differ, '*')
+ } else {
+ c.matches.Add(1)
+ c.report(src, c.opt.Match, '=')
+ if noHash {
+ c.noHashes.Add(1)
+ fs.Debugf(dstX, "OK - could not check hash")
+ } else {
+ fs.Debugf(dstX, "OK")
+ }
+ }
+ }()
+ } else {
+ err := fmt.Errorf("is file on %v but directory on %v", c.opt.Fsrc, c.opt.Fdst)
+ fs.Errorf(src, "%v", err)
+ _ = fs.CountError(ctx, err)
+ c.differences.Add(1)
+ c.dstFilesMissing.Add(1)
+ c.report(src, c.opt.MissingOnDst, '+')
+ }
+ case fs.Directory:
+ // Do the same thing to the entire contents of the directory
+ _, ok := dst.(fs.Directory)
+ if ok {
+ return true
+ }
+ err := fmt.Errorf("is file on %v but directory on %v", c.opt.Fdst, c.opt.Fsrc)
+ fs.Errorf(dst, "%v", err)
+ _ = fs.CountError(ctx, err)
+ c.differences.Add(1)
+ c.srcFilesMissing.Add(1)
+ c.report(dst, c.opt.MissingOnSrc, '-')
+
+ default:
+ panic("Bad object in DirEntries")
+ }
+ return false
+}
+
+// CheckFn checks the files in fsrc and fdst according to Size and
+// hash using checkFunction on each file to check the hashes.
+//
+// checkFunction sees if dst and src are identical
+//
+// it returns true if differences were found
+// it also returns whether it couldn't be hashed
+func CheckFn(ctx context.Context, opt *CheckOpt) error {
+ ci := fs.GetConfig(ctx)
+ if opt.Check == nil {
+ return errors.New("internal error: nil check function")
+ }
+ c := &checkMarch{
+ ctx: ctx,
+ tokens: make(chan struct{}, ci.Checkers),
+ opt: *opt,
+ }
+
+ // set up a march over fdst and fsrc
+ m := &march.March{
+ Ctx: ctx,
+ Fdst: c.opt.Fdst,
+ Fsrc: c.opt.Fsrc,
+ Dir: "",
+ Callback: c,
+ NoTraverse: ci.NoTraverse,
+ NoUnicodeNormalization: ci.NoUnicodeNormalization,
+ }
+ fs.Debugf(c.opt.Fdst, "Waiting for checks to finish")
+ err := m.Run(ctx)
+ c.wg.Wait() // wait for background go-routines
+
+ return c.reportResults(ctx, err)
+}
+
+func (c *checkMarch) reportResults(ctx context.Context, err error) error {
+ if c.dstFilesMissing.Load() > 0 {
+ fs.Logf(c.opt.Fdst, "%d files missing", c.dstFilesMissing.Load())
+ }
+ if c.srcFilesMissing.Load() > 0 {
+ entity := "files"
+ if c.opt.Fsrc == nil {
+ entity = "hashes"
+ }
+ fs.Logf(c.opt.Fsrc, "%d %s missing", c.srcFilesMissing.Load(), entity)
+ }
+
+ fs.Logf(c.opt.Fdst, "%d differences found", c.differences.Load())
+ if errs := accounting.Stats(ctx).GetErrors(); errs > 0 {
+ fs.Logf(c.opt.Fdst, "%d errors while checking", errs)
+ }
+ if c.noHashes.Load() > 0 {
+ fs.Logf(c.opt.Fdst, "%d hashes could not be checked", c.noHashes.Load())
+ }
+ if c.matches.Load() > 0 {
+ fs.Logf(c.opt.Fdst, "%d matching files", c.matches.Load())
+ }
+ if err != nil {
+ return err
+ }
+ if c.differences.Load() > 0 {
+ // Return an already counted error so we don't double count this error too
+ err = fserrors.FsError(fmt.Errorf("%d differences found", c.differences.Load()))
+ fserrors.Count(err)
+ return err
+ }
+ return nil
+}
+
+// Check the files in fsrc and fdst according to Size and hash
+func Check(ctx context.Context, opt *CheckOpt) error {
+ optCopy := *opt
+ optCopy.Check = func(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
+ same, ht, err := CheckHashes(ctx, src, dst)
+ if err != nil {
+ return true, false, err
+ }
+ if ht == hash.None {
+ return false, true, nil
+ }
+ if !same {
+ err = fmt.Errorf("%v differ", ht)
+ fs.Errorf(src, "%v", err)
+ return true, false, nil
+ }
+ return false, false, nil
+ }
+
+ return CheckFn(ctx, &optCopy)
+}
+
+// CheckEqualReaders checks to see if in1 and in2 have the same
+// content when read.
+//
+// it returns true if no differences were found
+func CheckEqualReaders(in1, in2 io.Reader) (equal bool, err error) {
+ const bufSize = 64 * 1024
+ buf1 := make([]byte, bufSize)
+ buf2 := make([]byte, bufSize)
+ for {
+ n1, err1 := readers.ReadFill(in1, buf1)
+ n2, err2 := readers.ReadFill(in2, buf2)
+ // check errors
+ if err1 != nil && err1 != io.EOF {
+ return false, err1
+ } else if err2 != nil && err2 != io.EOF {
+ return false, err2
+ }
+ // err1 && err2 are nil or io.EOF here
+ // process the data
+ if n1 != n2 || !bytes.Equal(buf1[:n1], buf2[:n2]) {
+ return false, nil
+ }
+ // if both streams finished the we have finished
+ if err1 == io.EOF && err2 == io.EOF {
+ break
+ }
+ }
+ return true, nil
+}
+
+// CheckIdenticalDownload checks to see if dst and src are identical
+// by reading all their bytes if necessary.
+//
+// it returns true if no differences were found
+func CheckIdenticalDownload(ctx context.Context, src, dst fs.Object) (equal bool, err error) {
+ ci := fs.GetConfig(ctx)
+ err = Retry(ctx, src, ci.LowLevelRetries, func() error {
+ equal, err = checkIdenticalDownload(ctx, src, dst)
+ return err
+ })
+ return equal, err
+}
+
+// Does the work for CheckIdenticalDownload
+func checkIdenticalDownload(ctx context.Context, src, dst fs.Object) (equal bool, err error) {
+ var in1, in2 io.ReadCloser
+ in1, err = Open(ctx, dst)
+ if err != nil {
+ return false, fmt.Errorf("failed to open %q: %w", dst, err)
+ }
+ tr1 := accounting.Stats(ctx).NewTransfer(dst, nil)
+ defer func() {
+ tr1.Done(ctx, nil) // error handling is done by the caller
+ }()
+ in1 = tr1.Account(ctx, in1).WithBuffer() // account and buffer the transfer
+
+ in2, err = Open(ctx, src)
+ if err != nil {
+ return false, fmt.Errorf("failed to open %q: %w", src, err)
+ }
+ tr2 := accounting.Stats(ctx).NewTransfer(dst, nil)
+ defer func() {
+ tr2.Done(ctx, nil) // error handling is done by the caller
+ }()
+ in2 = tr2.Account(ctx, in2).WithBuffer() // account and buffer the transfer
+
+ // To assign err variable before defer.
+ equal, err = CheckEqualReaders(in1, in2)
+ return
+}
+
+// CheckDownload checks the files in fsrc and fdst according to Size
+// and the actual contents of the files.
+func CheckDownload(ctx context.Context, opt *CheckOpt) error {
+ optCopy := *opt
+ optCopy.Check = func(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
+ same, err := CheckIdenticalDownload(ctx, src, dst)
+ if err != nil {
+ return true, true, fmt.Errorf("failed to download: %w", err)
+ }
+ if !same {
+ err = errors.New("contents differ")
+ fs.Errorf(src, "%v", err)
+ return true, false, nil
+ }
+ return false, false, nil
+ }
+ return CheckFn(ctx, &optCopy)
+}
+
+// ApplyTransforms handles --no-unicode-normalization and --ignore-case-sync for CheckSum
+// so that it matches behavior of Check (where it's handled by March)
+func ApplyTransforms(ctx context.Context, s string) string {
+ ci := fs.GetConfig(ctx)
+ return ToNormal(s, !ci.NoUnicodeNormalization, ci.IgnoreCaseSync)
+}
+
+// ToNormal normalizes case and unicode form and returns the transformed string.
+// It is similar to ApplyTransforms but does not use a context.
+// If normUnicode == true, s will be transformed to NFC.
+// If normCase == true, s will be transformed to lowercase.
+// If both are true, both transformations will be performed.
+func ToNormal(s string, normUnicode, normCase bool) string {
+ if normUnicode {
+ s = norm.NFC.String(s)
+ }
+ if normCase {
+ s = strings.ToLower(s)
+ }
+ return s
+}
+
+// CheckSum checks filesystem hashes against a SUM file
+func CheckSum(ctx context.Context, fsrc, fsum fs.Fs, sumFile string, hashType hash.Type, opt *CheckOpt, download bool) error {
+ var options CheckOpt
+ if opt != nil {
+ options = *opt
+ } else {
+ // default options for hashsum -c
+ options.Combined = os.Stdout
+ }
+ // CheckSum treats Fsrc and Fdst specially:
+ options.Fsrc = nil // no file system here, corresponds to the sum list
+ options.Fdst = fsrc // denotes the file system to check
+ opt = &options // override supplied argument
+
+ if !download && (hashType == hash.None || !opt.Fdst.Hashes().Contains(hashType)) {
+ return fmt.Errorf("%s: hash type is not supported by file system: %s", hashType, opt.Fdst)
+ }
+
+ if sumFile == "" {
+ return fmt.Errorf("not a sum file: %s", fsum)
+ }
+ sumObj, err := fsum.NewObject(ctx, sumFile)
+ if err != nil {
+ return fmt.Errorf("cannot open sum file: %w", err)
+ }
+ hashes, err := ParseSumFile(ctx, sumObj)
+ if err != nil {
+ return fmt.Errorf("failed to parse sum file: %w", err)
+ }
+
+ ci := fs.GetConfig(ctx)
+ c := &checkMarch{
+ ctx: ctx,
+ tokens: make(chan struct{}, ci.Checkers),
+ opt: *opt,
+ }
+ lastErr := ListFn(ctx, opt.Fdst, func(obj fs.Object) {
+ c.checkSum(ctx, obj, download, hashes, hashType)
+ })
+ c.wg.Wait() // wait for background go-routines
+
+ // make census of unhandled sums
+ fi := filter.GetConfig(ctx)
+ for filename, hash := range hashes {
+ if hash == "" { // the sum has been successfully consumed
+ continue
+ }
+ if !fi.IncludeRemote(filename) { // the file was filtered out
+ continue
+ }
+ // filesystem missed the file, sum wasn't consumed
+ err := fmt.Errorf("file not in %v", opt.Fdst)
+ fs.Errorf(filename, "%v", err)
+ _ = fs.CountError(ctx, err)
+ if lastErr == nil {
+ lastErr = err
+ }
+ c.dstFilesMissing.Add(1)
+ c.reportFilename(filename, opt.MissingOnDst, '+')
+ }
+
+ return c.reportResults(ctx, lastErr)
+}
+
+// checkSum checks single object against golden hashes
+func (c *checkMarch) checkSum(ctx context.Context, obj fs.Object, download bool, hashes HashSums, hashType hash.Type) {
+ normalizedRemote := ApplyTransforms(ctx, obj.Remote())
+ c.ioMu.Lock()
+ sumHash, sumFound := hashes[normalizedRemote]
+ hashes[normalizedRemote] = "" // mark sum as consumed
+ c.ioMu.Unlock()
+
+ if !sumFound && c.opt.OneWay {
+ return
+ }
+
+ var err error
+ tr := accounting.Stats(ctx).NewCheckingTransfer(obj, "hashing")
+ defer tr.Done(ctx, err)
+
+ if !sumFound {
+ err = errors.New("sum not found")
+ _ = fs.CountError(ctx, err)
+ fs.Errorf(obj, "%v", err)
+ c.differences.Add(1)
+ c.srcFilesMissing.Add(1)
+ c.report(obj, c.opt.MissingOnSrc, '-')
+ return
+ }
+
+ if !download {
+ var objHash string
+ objHash, err = obj.Hash(ctx, hashType)
+ c.matchSum(ctx, sumHash, objHash, obj, err, hashType)
+ return
+ }
+
+ c.wg.Add(1)
+ c.tokens <- struct{}{} // put a token to limit concurrency
+ go func() {
+ var (
+ objHash string
+ err error
+ in io.ReadCloser
+ )
+ defer func() {
+ c.matchSum(ctx, sumHash, objHash, obj, err, hashType)
+ <-c.tokens // get the token back to free up a slot
+ c.wg.Done()
+ }()
+ if in, err = Open(ctx, obj); err != nil {
+ return
+ }
+ tr := accounting.Stats(ctx).NewTransfer(obj, nil)
+ in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer
+ defer func() {
+ tr.Done(ctx, nil) // will close the stream
+ }()
+ hashVals, err2 := hash.StreamTypes(in, hash.NewHashSet(hashType))
+ if err2 != nil {
+ err = err2 // pass to matchSum
+ return
+ }
+ objHash = hashVals[hashType]
+ }()
+}
+
+// matchSum sums up the results of hashsum matching for an object
+func (c *checkMarch) matchSum(ctx context.Context, sumHash, objHash string, obj fs.Object, err error, hashType hash.Type) {
+ switch {
+ case err != nil:
+ _ = fs.CountError(ctx, err)
+ fs.Errorf(obj, "Failed to calculate hash: %v", err)
+ c.report(obj, c.opt.Error, '!')
+ case sumHash == "":
+ err = errors.New("duplicate file")
+ _ = fs.CountError(ctx, err)
+ fs.Errorf(obj, "%v", err)
+ c.report(obj, c.opt.Error, '!')
+ case objHash == "":
+ fs.Debugf(nil, "%v = %s (sum)", hashType, sumHash)
+ fs.Debugf(obj, "%v - could not check hash (%v)", hashType, c.opt.Fdst)
+ c.noHashes.Add(1)
+ c.matches.Add(1)
+ c.report(obj, c.opt.Match, '=')
+ case objHash == sumHash:
+ fs.Debugf(obj, "%v = %s OK", hashType, sumHash)
+ c.matches.Add(1)
+ c.report(obj, c.opt.Match, '=')
+ default:
+ err = errors.New("files differ")
+ _ = fs.CountError(ctx, err)
+ fs.Debugf(nil, "%v = %s (sum)", hashType, sumHash)
+ fs.Debugf(obj, "%v = %s (%v)", hashType, objHash, c.opt.Fdst)
+ fs.Errorf(obj, "%v", err)
+ c.differences.Add(1)
+ c.report(obj, c.opt.Differ, '*')
+ }
+}
+
+// HashSums represents a parsed SUM file
+type HashSums map[string]string
+
+// ParseSumFile parses a hash SUM file and returns hashes as a map
+func ParseSumFile(ctx context.Context, sumFile fs.Object) (HashSums, error) {
+ rd, err := Open(ctx, sumFile)
+ if err != nil {
+ return nil, err
+ }
+ parser := bufio.NewReader(rd)
+
+ const maxWarn = 3
+ numWarn := 0
+
+ re := regexp.MustCompile(`^([^ ]+) [ *](.+)$`)
+ hashes := HashSums{}
+ for lineNo := 0; true; lineNo++ {
+ lineBytes, _, err := parser.ReadLine()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+ line := string(lineBytes)
+ if line == "" {
+ continue
+ }
+
+ fields := re.FindStringSubmatch(ApplyTransforms(ctx, line))
+ if fields == nil {
+ numWarn++
+ if numWarn <= maxWarn {
+ fs.Logf(sumFile, "improperly formatted checksum line %d", lineNo)
+ }
+ continue
+ }
+
+ sum, file := fields[1], fields[2]
+ if hashes[file] != "" {
+ numWarn++
+ if numWarn <= maxWarn {
+ fs.Logf(sumFile, "duplicate file on checksum line %d", lineNo)
+ }
+ continue
+ }
+
+ // We've standardised on lower case checksums in rclone internals.
+ hashes[file] = strings.ToLower(sum)
+ }
+
+ if numWarn > maxWarn {
+ fs.Logf(sumFile, "%d warning(s) suppressed...", numWarn-maxWarn)
+ }
+ if err = rd.Close(); err != nil {
+ return nil, err
+ }
+ return hashes, nil
+}
diff --git a/fs/operations/check_test.go b/fs/operations/check_test.go
new file mode 100644
index 0000000..e6c3092
--- /dev/null
+++ b/fs/operations/check_test.go
@@ -0,0 +1,617 @@
+package operations_test
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "sort"
+ "strings"
+ "testing"
+
+ "github.com/rclone/rclone/cmd/bisync/bilib"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/fstest"
+ "github.com/rclone/rclone/lib/readers"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "golang.org/x/text/unicode/norm"
+)
+
+func testCheck(t *testing.T, checkFunction func(ctx context.Context, opt *operations.CheckOpt) error) {
+ r := fstest.NewRun(t)
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
+
+ addBuffers := func(opt *operations.CheckOpt) {
+ opt.Combined = new(bytes.Buffer)
+ opt.MissingOnSrc = new(bytes.Buffer)
+ opt.MissingOnDst = new(bytes.Buffer)
+ opt.Match = new(bytes.Buffer)
+ opt.Differ = new(bytes.Buffer)
+ opt.Error = new(bytes.Buffer)
+ }
+
+ sortLines := func(in string) []string {
+ if in == "" {
+ return []string{}
+ }
+ lines := strings.Split(in, "\n")
+ sort.Strings(lines)
+ return lines
+ }
+
+ checkBuffer := func(name string, want map[string]string, out io.Writer) {
+ expected := want[name]
+ buf, ok := out.(*bytes.Buffer)
+ require.True(t, ok)
+ assert.Equal(t, sortLines(expected), sortLines(buf.String()), name)
+ }
+
+ checkBuffers := func(opt *operations.CheckOpt, want map[string]string) {
+ checkBuffer("combined", want, opt.Combined)
+ checkBuffer("missingonsrc", want, opt.MissingOnSrc)
+ checkBuffer("missingondst", want, opt.MissingOnDst)
+ checkBuffer("match", want, opt.Match)
+ checkBuffer("differ", want, opt.Differ)
+ checkBuffer("error", want, opt.Error)
+ }
+
+ check := func(i int, wantErrors int64, wantChecks int64, oneway bool, wantOutput map[string]string) {
+ t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+ accounting.GlobalStats().ResetCounters()
+ opt := operations.CheckOpt{
+ Fdst: r.Fremote,
+ Fsrc: r.Flocal,
+ OneWay: oneway,
+ }
+ addBuffers(&opt)
+ var err error
+ buf := bilib.CaptureOutput(func() {
+ err = checkFunction(ctx, &opt)
+ })
+ gotErrors := accounting.GlobalStats().GetErrors()
+ gotChecks := accounting.GlobalStats().GetChecks()
+ if wantErrors == 0 && err != nil {
+ t.Errorf("%d: Got error when not expecting one: %v", i, err)
+ }
+ if wantErrors != 0 && err == nil {
+ t.Errorf("%d: No error when expecting one", i)
+ }
+ if wantErrors != gotErrors {
+ t.Errorf("%d: Expecting %d errors but got %d", i, wantErrors, gotErrors)
+ }
+ if gotChecks > 0 && !strings.Contains(string(buf), "matching files") {
+ t.Errorf("%d: Total files matching line missing", i)
+ }
+ if wantChecks != gotChecks {
+ t.Errorf("%d: Expecting %d total matching files but got %d", i, wantChecks, gotChecks)
+ }
+ checkBuffers(&opt, wantOutput)
+ })
+ }
+
+ file1 := r.WriteBoth(ctx, "rutabaga", "is tasty", t3)
+ r.CheckRemoteItems(t, file1)
+ r.CheckLocalItems(t, file1)
+ check(1, 0, 1, false, map[string]string{
+ "combined": "= rutabaga\n",
+ "missingonsrc": "",
+ "missingondst": "",
+ "match": "rutabaga\n",
+ "differ": "",
+ "error": "",
+ })
+
+ file2 := r.WriteFile("potato2", "------------------------------------------------------------", t1)
+ r.CheckLocalItems(t, file1, file2)
+ check(2, 1, 1, false, map[string]string{
+ "combined": "+ potato2\n= rutabaga\n",
+ "missingonsrc": "",
+ "missingondst": "potato2\n",
+ "match": "rutabaga\n",
+ "differ": "",
+ "error": "",
+ })
+
+ file3 := r.WriteObject(ctx, "empty space", "-", t2)
+ r.CheckRemoteItems(t, file1, file3)
+ check(3, 2, 1, false, map[string]string{
+ "combined": "- empty space\n+ potato2\n= rutabaga\n",
+ "missingonsrc": "empty space\n",
+ "missingondst": "potato2\n",
+ "match": "rutabaga\n",
+ "differ": "",
+ "error": "",
+ })
+
+ file2r := file2
+ if ci.SizeOnly {
+ file2r = r.WriteObject(ctx, "potato2", "--Some-Differences-But-Size-Only-Is-Enabled-----------------", t1)
+ } else {
+ r.WriteObject(ctx, "potato2", "------------------------------------------------------------", t1)
+ }
+ r.CheckRemoteItems(t, file1, file2r, file3)
+ check(4, 1, 2, false, map[string]string{
+ "combined": "- empty space\n= potato2\n= rutabaga\n",
+ "missingonsrc": "empty space\n",
+ "missingondst": "",
+ "match": "rutabaga\npotato2\n",
+ "differ": "",
+ "error": "",
+ })
+
+ file3r := file3
+ file3l := r.WriteFile("empty space", "DIFFER", t2)
+ r.CheckLocalItems(t, file1, file2, file3l)
+ check(5, 1, 3, false, map[string]string{
+ "combined": "* empty space\n= potato2\n= rutabaga\n",
+ "missingonsrc": "",
+ "missingondst": "",
+ "match": "potato2\nrutabaga\n",
+ "differ": "empty space\n",
+ "error": "",
+ })
+
+ file4 := r.WriteObject(ctx, "remotepotato", "------------------------------------------------------------", t1)
+ r.CheckRemoteItems(t, file1, file2r, file3r, file4)
+ check(6, 2, 3, false, map[string]string{
+ "combined": "* empty space\n= potato2\n= rutabaga\n- remotepotato\n",
+ "missingonsrc": "remotepotato\n",
+ "missingondst": "",
+ "match": "potato2\nrutabaga\n",
+ "differ": "empty space\n",
+ "error": "",
+ })
+ check(7, 1, 3, true, map[string]string{
+ "combined": "* empty space\n= potato2\n= rutabaga\n",
+ "missingonsrc": "",
+ "missingondst": "",
+ "match": "potato2\nrutabaga\n",
+ "differ": "empty space\n",
+ "error": "",
+ })
+}
+
+func TestCheck(t *testing.T) {
+ testCheck(t, operations.Check)
+}
+
+func TestCheckFsError(t *testing.T) {
+ ctx := context.Background()
+ dstFs, err := fs.NewFs(ctx, "nonexistent")
+ if err != nil {
+ t.Fatal(err)
+ }
+ srcFs, err := fs.NewFs(ctx, "nonexistent")
+ if err != nil {
+ t.Fatal(err)
+ }
+ opt := operations.CheckOpt{
+ Fdst: dstFs,
+ Fsrc: srcFs,
+ OneWay: false,
+ }
+ err = operations.Check(ctx, &opt)
+ require.Error(t, err)
+}
+
+func TestCheckDownload(t *testing.T) {
+ testCheck(t, operations.CheckDownload)
+}
+
+func TestCheckSizeOnly(t *testing.T) {
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
+ ci.SizeOnly = true
+ defer func() { ci.SizeOnly = false }()
+ TestCheck(t)
+}
+
+func TestCheckEqualReaders(t *testing.T) {
+ b65a := make([]byte, 65*1024)
+ b65b := make([]byte, 65*1024)
+ b65b[len(b65b)-1] = 1
+ b66 := make([]byte, 66*1024)
+
+ equal, err := operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b65a))
+ assert.NoError(t, err)
+ assert.Equal(t, equal, true)
+
+ equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b65b))
+ assert.NoError(t, err)
+ assert.Equal(t, equal, false)
+
+ equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b66))
+ assert.NoError(t, err)
+ assert.Equal(t, equal, false)
+
+ equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b66), bytes.NewBuffer(b65a))
+ assert.NoError(t, err)
+ assert.Equal(t, equal, false)
+
+ myErr := errors.New("sentinel")
+ wrap := func(b []byte) io.Reader {
+ r := bytes.NewBuffer(b)
+ e := readers.ErrorReader{Err: myErr}
+ return io.MultiReader(r, e)
+ }
+
+ equal, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b65a))
+ assert.Equal(t, myErr, err)
+ assert.Equal(t, equal, false)
+
+ equal, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b65b))
+ assert.Equal(t, myErr, err)
+ assert.Equal(t, equal, false)
+
+ equal, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b66))
+ assert.Equal(t, myErr, err)
+ assert.Equal(t, equal, false)
+
+ equal, err = operations.CheckEqualReaders(wrap(b66), bytes.NewBuffer(b65a))
+ assert.Equal(t, myErr, err)
+ assert.Equal(t, equal, false)
+
+ equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b65a))
+ assert.Equal(t, myErr, err)
+ assert.Equal(t, equal, false)
+
+ equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b65b))
+ assert.Equal(t, myErr, err)
+ assert.Equal(t, equal, false)
+
+ equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b66))
+ assert.Equal(t, myErr, err)
+ assert.Equal(t, equal, false)
+
+ equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b66), wrap(b65a))
+ assert.Equal(t, myErr, err)
+ assert.Equal(t, equal, false)
+}
+
+func TestParseSumFile(t *testing.T) {
+ r := fstest.NewRun(t)
+ ctx := context.Background()
+
+ const sumFile = "test.sum"
+
+ samples := []struct {
+ hash, sep, name string
+ ok bool
+ }{
+ {"1", " ", "file1", true},
+ {"2", " *", "file2", true},
+ {"3", " ", " file3 ", true},
+ {"4", " ", "\tfile3\t", true},
+ {"5", " ", "file5", false},
+ {"6", "\t", "file6", false},
+ {"7", " \t", " file7 ", false},
+ {"", " ", "file8", false},
+ {"", "", "file9", false},
+ }
+
+ for _, eol := range []string{"\n", "\r\n"} {
+ data := &bytes.Buffer{}
+ wantNum := 0
+ for _, s := range samples {
+ _, _ = data.WriteString(s.hash + s.sep + s.name + eol)
+ if s.ok {
+ wantNum++
+ }
+ }
+
+ _ = r.WriteObject(ctx, sumFile, data.String(), t1)
+ file := fstest.NewObject(ctx, t, r.Fremote, sumFile)
+ sums, err := operations.ParseSumFile(ctx, file)
+ assert.NoError(t, err)
+
+ assert.Equal(t, wantNum, len(sums))
+ for _, s := range samples {
+ if s.ok {
+ assert.Equal(t, s.hash, sums[s.name])
+ }
+ }
+ }
+}
+
+func testCheckSum(t *testing.T, download bool) {
+ const dataDir = "data"
+ const sumFile = "test.sum"
+
+ hashType := hash.MD5
+ const (
+ testString1 = "Hello, World!"
+ testDigest1 = "65a8e27d8879283831b664bd8b7f0ad4"
+ testDigest1Upper = "65A8E27D8879283831B664BD8B7F0AD4"
+ testString2 = "I am the walrus"
+ testDigest2 = "87396e030ef3f5b35bbf85c0a09a4fb3"
+ testDigest2Mixed = "87396e030EF3f5b35BBf85c0a09a4FB3"
+ )
+
+ type wantType map[string]string
+
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+
+ subRemote := r.FremoteName
+ if !strings.HasSuffix(subRemote, ":") {
+ subRemote += "/"
+ }
+ subRemote += dataDir
+ dataFs, err := fs.NewFs(ctx, subRemote)
+ require.NoError(t, err)
+
+ if !download && !dataFs.Hashes().Contains(hashType) {
+ t.Skipf("%s lacks %s, skipping", dataFs, hashType)
+ }
+
+ makeFile := func(name, content string) fstest.Item {
+ remote := dataDir + "/" + name
+ return r.WriteObject(ctx, remote, content, t1)
+ }
+
+ makeSums := func(sums operations.HashSums) fstest.Item {
+ files := make([]string, 0, len(sums))
+ for name := range sums {
+ files = append(files, name)
+ }
+ sort.Strings(files)
+ buf := &bytes.Buffer{}
+ for _, name := range files {
+ _, _ = fmt.Fprintf(buf, "%s %s\n", sums[name], name)
+ }
+ return r.WriteObject(ctx, sumFile, buf.String(), t1)
+ }
+
+ sortLines := func(in string) []string {
+ if in == "" {
+ return []string{}
+ }
+ lines := strings.Split(in, "\n")
+ sort.Strings(lines)
+ return lines
+ }
+
+ checkResult := func(runNo int, want wantType, name string, out io.Writer) {
+ expected := want[name]
+ buf, ok := out.(*bytes.Buffer)
+ require.True(t, ok)
+ assert.Equal(t, sortLines(expected), sortLines(buf.String()), "wrong %s result in run %d", name, runNo)
+ }
+
+ checkRun := func(runNo, wantChecks, wantErrors int, want wantType) {
+ accounting.GlobalStats().ResetCounters()
+
+ opt := operations.CheckOpt{
+ Combined: new(bytes.Buffer),
+ Match: new(bytes.Buffer),
+ Differ: new(bytes.Buffer),
+ Error: new(bytes.Buffer),
+ MissingOnSrc: new(bytes.Buffer),
+ MissingOnDst: new(bytes.Buffer),
+ }
+ var err error
+ buf := bilib.CaptureOutput(func() {
+ err = operations.CheckSum(ctx, dataFs, r.Fremote, sumFile, hashType, &opt, download)
+ })
+ gotErrors := int(accounting.GlobalStats().GetErrors())
+ if wantErrors == 0 {
+ assert.NoError(t, err, "unexpected error in run %d", runNo)
+ }
+ if wantErrors > 0 {
+ assert.Error(t, err, "no expected error in run %d", runNo)
+ }
+ assert.Equal(t, wantErrors, gotErrors, "wrong error count in run %d", runNo)
+
+ gotChecks := int(accounting.GlobalStats().GetChecks())
+ if wantChecks > 0 || gotChecks > 0 {
+ assert.Contains(t, string(buf), "matching files", "missing matching files in run %d", runNo)
+ }
+ assert.Equal(t, wantChecks, gotChecks, "wrong number of checks in run %d", runNo)
+
+ checkResult(runNo, want, "combined", opt.Combined)
+ checkResult(runNo, want, "missingonsrc", opt.MissingOnSrc)
+ checkResult(runNo, want, "missingondst", opt.MissingOnDst)
+ checkResult(runNo, want, "match", opt.Match)
+ checkResult(runNo, want, "differ", opt.Differ)
+ checkResult(runNo, want, "error", opt.Error)
+ }
+
+ check := func(runNo, wantChecks, wantErrors int, wantResults wantType) {
+ runName := fmt.Sprintf("subtest%d", runNo)
+ t.Run(runName, func(t *testing.T) {
+ checkRun(runNo, wantChecks, wantErrors, wantResults)
+ })
+ }
+
+ file1 := makeFile("banana", testString1)
+ fcsums := makeSums(operations.HashSums{
+ "banana": testDigest1,
+ })
+ r.CheckRemoteItems(t, fcsums, file1)
+ check(1, 1, 0, wantType{
+ "combined": "= banana\n",
+ "missingonsrc": "",
+ "missingondst": "",
+ "match": "banana\n",
+ "differ": "",
+ "error": "",
+ })
+
+ file2 := makeFile("potato", testString2)
+ fcsums = makeSums(operations.HashSums{
+ "banana": testDigest1,
+ })
+ r.CheckRemoteItems(t, fcsums, file1, file2)
+ check(2, 2, 1, wantType{
+ "combined": "- potato\n= banana\n",
+ "missingonsrc": "potato\n",
+ "missingondst": "",
+ "match": "banana\n",
+ "differ": "",
+ "error": "",
+ })
+
+ fcsums = makeSums(operations.HashSums{
+ "banana": testDigest1,
+ "potato": testDigest2,
+ })
+ r.CheckRemoteItems(t, fcsums, file1, file2)
+ check(3, 2, 0, wantType{
+ "combined": "= potato\n= banana\n",
+ "missingonsrc": "",
+ "missingondst": "",
+ "match": "banana\npotato\n",
+ "differ": "",
+ "error": "",
+ })
+
+ fcsums = makeSums(operations.HashSums{
+ "banana": testDigest2,
+ "potato": testDigest2,
+ })
+ r.CheckRemoteItems(t, fcsums, file1, file2)
+ check(4, 2, 1, wantType{
+ "combined": "* banana\n= potato\n",
+ "missingonsrc": "",
+ "missingondst": "",
+ "match": "potato\n",
+ "differ": "banana\n",
+ "error": "",
+ })
+
+ fcsums = makeSums(operations.HashSums{
+ "banana": testDigest1,
+ "potato": testDigest2,
+ "orange": testDigest2,
+ })
+ r.CheckRemoteItems(t, fcsums, file1, file2)
+ check(5, 2, 1, wantType{
+ "combined": "+ orange\n= potato\n= banana\n",
+ "missingonsrc": "",
+ "missingondst": "orange\n",
+ "match": "banana\npotato\n",
+ "differ": "",
+ "error": "",
+ })
+
+ fcsums = makeSums(operations.HashSums{
+ "banana": testDigest1,
+ "potato": testDigest1,
+ "orange": testDigest2,
+ })
+ r.CheckRemoteItems(t, fcsums, file1, file2)
+ check(6, 2, 2, wantType{
+ "combined": "+ orange\n* potato\n= banana\n",
+ "missingonsrc": "",
+ "missingondst": "orange\n",
+ "match": "banana\n",
+ "differ": "potato\n",
+ "error": "",
+ })
+
+ // test mixed-case checksums
+ file1 = makeFile("banana", testString1)
+ file2 = makeFile("potato", testString2)
+ fcsums = makeSums(operations.HashSums{
+ "banana": testDigest1Upper,
+ "potato": testDigest2Mixed,
+ })
+ r.CheckRemoteItems(t, fcsums, file1, file2)
+ check(7, 2, 0, wantType{
+ "combined": "= banana\n= potato\n",
+ "missingonsrc": "",
+ "missingondst": "",
+ "match": "banana\npotato\n",
+ "differ": "",
+ "error": "",
+ })
+}
+
+func TestCheckSum(t *testing.T) {
+ testCheckSum(t, false)
+}
+
+func TestCheckSumDownload(t *testing.T) {
+ testCheckSum(t, true)
+}
+
+func TestApplyTransforms(t *testing.T) {
+ var (
+ hashType = hash.MD5
+ content = "Hello, World!"
+ hash = "65a8e27d8879283831b664bd8b7f0ad4"
+ nfc = norm.NFC.String(norm.NFD.String("測試_Русский___ě_áñ"))
+ nfd = norm.NFD.String(nfc)
+ nfcx2 = nfc + nfc
+ nfdx2 = nfd + nfd
+ both = nfc + nfd
+ upper = "HELLO, WORLD!"
+ lower = "hello, world!"
+ upperlowermixed = "HeLlO, wOrLd!"
+ )
+
+ testScenario := func(checkfileName, remotefileName, scenario string) {
+ r := fstest.NewRunIndividual(t)
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
+ opt := operations.CheckOpt{}
+
+ remotefile := r.WriteObject(ctx, remotefileName, content, t2)
+ // test whether remote is capable of running test
+ entries, err := r.Fremote.List(ctx, "")
+ assert.NoError(t, err)
+ if entries.Len() == 1 && entries[0].Remote() != remotefileName {
+ t.Skipf("Fs is incapable of running test, skipping: %s (expected: %s (%s) actual: %s (%s))", scenario, remotefileName, detectEncoding(remotefileName), entries[0].Remote(), detectEncoding(entries[0].Remote()))
+ }
+
+ checkfile := r.WriteFile("test.sum", hash+" "+checkfileName, t2)
+ r.CheckLocalItems(t, checkfile)
+ assert.False(t, checkfileName == remotefile.Path, "Values match but should not: %s %s", checkfileName, remotefile.Path)
+
+ testname := scenario + " (without normalization)"
+ println(testname)
+ ci.NoUnicodeNormalization = true
+ ci.IgnoreCaseSync = false
+ accounting.GlobalStats().ResetCounters()
+ err = operations.CheckSum(ctx, r.Fremote, r.Flocal, "test.sum", hashType, &opt, true)
+ assert.Error(t, err, "no expected error for %s %v %v", testname, checkfileName, remotefileName)
+
+ testname = scenario + " (with normalization)"
+ println(testname)
+ ci.NoUnicodeNormalization = false
+ ci.IgnoreCaseSync = true
+ accounting.GlobalStats().ResetCounters()
+ err = operations.CheckSum(ctx, r.Fremote, r.Flocal, "test.sum", hashType, &opt, true)
+ assert.NoError(t, err, "unexpected error for %s %v %v", testname, checkfileName, remotefileName)
+ }
+
+ testScenario(upper, lower, "upper checkfile vs. lower remote")
+ testScenario(lower, upper, "lower checkfile vs. upper remote")
+ testScenario(lower, upperlowermixed, "lower checkfile vs. upperlowermixed remote")
+ testScenario(upperlowermixed, upper, "upperlowermixed checkfile vs. upper remote")
+ testScenario(nfd, nfc, "NFD checkfile vs. NFC remote")
+ testScenario(nfc, nfd, "NFC checkfile vs. NFD remote")
+ testScenario(nfdx2, both, "NFDx2 checkfile vs. both remote")
+ testScenario(nfcx2, both, "NFCx2 checkfile vs. both remote")
+ testScenario(both, nfdx2, "both checkfile vs. NFDx2 remote")
+ testScenario(both, nfcx2, "both checkfile vs. NFCx2 remote")
+}
+
+func detectEncoding(s string) string {
+ if norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) {
+ return "BOTH"
+ }
+ if !norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) {
+ return "NFD"
+ }
+ if norm.NFC.IsNormalString(s) && !norm.NFD.IsNormalString(s) {
+ return "NFC"
+ }
+ return "OTHER"
+}
diff --git a/fs/operations/copy.go b/fs/operations/copy.go
new file mode 100644
index 0000000..f7871d2
--- /dev/null
+++ b/fs/operations/copy.go
@@ -0,0 +1,422 @@
+// This file implements operations.Copy
+//
+// This is probably the most important operation in rclone.
+
+package operations
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "hash/crc32"
+ "io"
+ "path"
+ "strings"
+ "time"
+ "unicode/utf8"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/fserrors"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/lib/atexit"
+ "github.com/rclone/rclone/lib/pacer"
+ "github.com/rclone/rclone/lib/transform"
+)
+
+// State of the copy
+type copy struct {
+ f fs.Fs // destination fs.Fs
+ dstFeatures *fs.Features // Features() for fs.Fs
+ dst fs.Object // destination object to update, may be nil
+ remote string // destination path, used if dst is nil
+ src fs.Object // source object
+ ci *fs.ConfigInfo // current config
+ maxTries int // max number of tries to do the copy
+ doUpdate bool // whether we are updating an existing file or not
+ hashType hash.Type // common hash to use
+ hashOption *fs.HashesOption // open option for the common hash
+ tr *accounting.Transfer // accounting for the transfer
+ inplace bool // set if we are updating inplace and not using a partial name
+ remoteForCopy string // the name used for the transfer, either remote or remote+".partial"
+}
+
+// Used to remove a failed copy
+func (c *copy) removeFailedCopy(ctx context.Context, o fs.Object) {
+ if o == nil {
+ return
+ }
+ fs.Infof(o, "Removing failed copy")
+ err := o.Remove(ctx)
+ if err != nil {
+ fs.Infof(o, "Failed to remove failed copy: %s", err)
+ }
+}
+
+// Used to remove a failed partial copy
+func (c *copy) removeFailedPartialCopy(ctx context.Context, f fs.Fs, remote string) {
+ o, err := f.NewObject(ctx, remote)
+ if errors.Is(err, fs.ErrorObjectNotFound) {
+ // Assume object has been deleted
+ return
+ }
+ if err != nil {
+ fs.Infof(remote, "Failed to remove failed partial copy: %s", err)
+ return
+ }
+ c.removeFailedCopy(ctx, o)
+}
+
+// TruncateString s to n bytes.
+//
+// If s is valid UTF-8 then this may truncate to fewer than n bytes to
+// make the returned string also valid UTF-8.
+func TruncateString(s string, n int) string {
+ truncated := s[:n]
+ if !utf8.ValidString(s) {
+ // If input string wasn't valid UTF-8 then just return the truncation
+ return truncated
+ }
+ for len(truncated) > 0 {
+ if utf8.ValidString(truncated) {
+ return truncated
+ }
+ // Remove 1 byte until valid
+ truncated = truncated[:len(truncated)-1]
+ }
+ return truncated
+}
+
+// Check to see if we should be using a partial name and return the name for the copy and the inplace flag
+func (c *copy) checkPartial(ctx context.Context) (remoteForCopy string, inplace bool, err error) {
+ remoteForCopy = c.remote
+ if c.ci.Inplace || c.dstFeatures.Move == nil || !c.dstFeatures.PartialUploads || strings.HasSuffix(c.remote, ".rclonelink") {
+ return remoteForCopy, true, nil
+ }
+ if len(c.ci.PartialSuffix) > 16 {
+ return remoteForCopy, true, fmt.Errorf("expecting length of --partial-suffix to be not greater than %d but got %d", 16, len(c.ci.PartialSuffix))
+ }
+ // Avoid making the leaf name longer if it's already lengthy to avoid
+ // trouble with file name length limits.
+
+ // generate a stable random suffix by hashing the filename and fingerprint
+ hasher := crc32.New(crc32.IEEETable)
+ _, _ = hasher.Write([]byte(c.remote))
+ _, _ = hasher.Write([]byte(fs.Fingerprint(ctx, c.src, true)))
+ hash := hasher.Sum32()
+
+ suffix := fmt.Sprintf(".%08x%s", hash, c.ci.PartialSuffix)
+ base := path.Base(remoteForCopy)
+ if len(base) > 100 {
+ remoteForCopy = TruncateString(remoteForCopy, len(remoteForCopy)-len(suffix)) + suffix
+ } else {
+ remoteForCopy += suffix
+ }
+ return remoteForCopy, false, nil
+}
+
+// Check to see if we have hit max transfer limits
+func (c *copy) checkLimits(ctx context.Context) (err error) {
+ if c.ci.MaxTransfer < 0 {
+ return nil
+ }
+ var bytesSoFar int64
+ if c.ci.CutoffMode == fs.CutoffModeCautious {
+ bytesSoFar = accounting.Stats(ctx).GetBytesWithPending() + c.src.Size()
+ } else {
+ bytesSoFar = accounting.Stats(ctx).GetBytes()
+ }
+ if bytesSoFar >= int64(c.ci.MaxTransfer) {
+ if c.ci.CutoffMode == fs.CutoffModeHard {
+ return accounting.ErrorMaxTransferLimitReachedFatal
+ }
+ return accounting.ErrorMaxTransferLimitReachedGraceful
+ }
+ return nil
+}
+
+// Server side copy c.src to (c.f, c.remoteForCopy) if possible or return fs.ErrorCantCopy if not
+func (c *copy) serverSideCopy(ctx context.Context) (actionTaken string, newDst fs.Object, err error) {
+ doCopy := c.dstFeatures.Copy
+ serverSideCopyOK := false
+ if doCopy == nil {
+ serverSideCopyOK = false
+ } else if SameConfig(c.src.Fs(), c.f) {
+ serverSideCopyOK = true
+ } else if SameRemoteType(c.src.Fs(), c.f) {
+ serverSideCopyOK = c.dstFeatures.ServerSideAcrossConfigs || c.ci.ServerSideAcrossConfigs
+ }
+ if !serverSideCopyOK {
+ return actionTaken, nil, fs.ErrorCantCopy
+ }
+ in := c.tr.Account(ctx, nil) // account the transfer
+ in.ServerSideTransferStart()
+ newDst, err = doCopy(ctx, c.src, c.remoteForCopy)
+ if err == nil {
+ in.ServerSideCopyEnd(newDst.Size()) // account the bytes for the server-side transfer
+ }
+ _ = in.Close()
+ if errors.Is(err, fs.ErrorCantCopy) {
+ c.tr.Reset(ctx) // skip incomplete accounting - will be overwritten by the manual copy
+ }
+ actionTaken = "Copied (server-side copy)"
+ return actionTaken, newDst, err
+}
+
+// Copy c.src to (c.f, c.remoteForCopy) using multiThreadCopy
+func (c *copy) multiThreadCopy(ctx context.Context, uploadOptions []fs.OpenOption) (actionTaken string, newDst fs.Object, err error) {
+ newDst, err = multiThreadCopy(ctx, c.f, c.remoteForCopy, c.src, c.ci.MultiThreadStreams, c.tr, uploadOptions...)
+ if c.doUpdate {
+ actionTaken = "Multi-thread Copied (replaced existing)"
+ } else {
+ actionTaken = "Multi-thread Copied (new)"
+ }
+ return actionTaken, newDst, err
+}
+
+// Copy the stream from in to (c.f, c.remoteForCopy) and close it
+//
+// Use Rcat to handle both remotes supporting and not supporting PutStream.
+func (c *copy) rcat(ctx context.Context, in io.ReadCloser) (actionTaken string, newDst fs.Object, err error) {
+ // Make any metadata to pass to rcat
+ var meta fs.Metadata
+ if c.ci.Metadata {
+ meta, err = fs.GetMetadata(ctx, c.src)
+ if err != nil {
+ fs.Errorf(c.src, "Failed to read metadata: %v", err)
+ }
+ }
+
+ // NB Rcat closes in0
+ fsrc, ok := c.src.Fs().(fs.Fs)
+ if !ok {
+ fsrc = nil
+ }
+ newDst, err = rcatSrc(ctx, c.f, c.remoteForCopy, in, c.src.ModTime(ctx), meta, fsrc)
+ if c.doUpdate {
+ actionTaken = "Copied (Rcat, replaced existing)"
+ } else {
+ actionTaken = "Copied (Rcat, new)"
+ }
+ return actionTaken, newDst, err
+}
+
+// Copy the stream from in to (c.f, c.remoteForCopy) and close it
+func (c *copy) updateOrPut(ctx context.Context, in io.ReadCloser, uploadOptions []fs.OpenOption) (actionTaken string, newDst fs.Object, err error) {
+ // account and buffer the transfer
+ inAcc := c.tr.Account(ctx, in).WithBuffer()
+ var wrappedSrc fs.ObjectInfo = c.src
+
+ // We try to pass the original object if possible
+ if c.src.Remote() != c.remoteForCopy {
+ wrappedSrc = fs.NewOverrideRemote(c.src, c.remoteForCopy)
+ }
+ if c.doUpdate && c.inplace {
+ err = c.dst.Update(ctx, inAcc, wrappedSrc, uploadOptions...)
+ // Make sure newDst is c.dst since we updated it
+ if err == nil {
+ newDst = c.dst
+ }
+ } else {
+ newDst, err = c.f.Put(ctx, inAcc, wrappedSrc, uploadOptions...)
+ }
+ closeErr := inAcc.Close()
+ if err == nil {
+ err = closeErr
+ }
+ if c.doUpdate {
+ actionTaken = "Copied (replaced existing)"
+ } else {
+ actionTaken = "Copied (new)"
+ }
+ return actionTaken, newDst, err
+}
+
+// Do a manual copy by reading the bytes and writing them
+func (c *copy) manualCopy(ctx context.Context) (actionTaken string, newDst fs.Object, err error) {
+ // Remove partial files on premature exit
+ if !c.inplace {
+ defer atexit.Unregister(atexit.Register(func() {
+ ctx := context.Background()
+ c.removeFailedPartialCopy(ctx, c.f, c.remoteForCopy)
+ }))
+ }
+
+ // Options for the upload
+ uploadOptions := []fs.OpenOption{c.hashOption}
+ for _, option := range c.ci.UploadHeaders {
+ uploadOptions = append(uploadOptions, option)
+ }
+ if c.ci.MetadataSet != nil {
+ uploadOptions = append(uploadOptions, fs.MetadataOption(c.ci.MetadataSet))
+ }
+
+ // Options for the download
+ downloadOptions := []fs.OpenOption{c.hashOption}
+ for _, option := range c.ci.DownloadHeaders {
+ downloadOptions = append(downloadOptions, option)
+ }
+
+ if doMultiThreadCopy(ctx, c.f, c.src) {
+ return c.multiThreadCopy(ctx, uploadOptions)
+ }
+
+ var in io.ReadCloser
+ in, err = Open(ctx, c.src, downloadOptions...)
+ if err != nil {
+ return actionTaken, nil, fmt.Errorf("failed to open source object: %w", err)
+ }
+
+ // Note that c.rcat and c.updateOrPut close in
+ if c.src.Size() == -1 {
+ return c.rcat(ctx, in)
+ }
+ return c.updateOrPut(ctx, in, uploadOptions)
+}
+
+// Verify the copy
+func (c *copy) verify(ctx context.Context, newDst fs.Object) (err error) {
+ // Verify sizes are the same after transfer
+ if sizeDiffers(ctx, c.src, newDst) {
+ return fmt.Errorf("corrupted on transfer: sizes differ src(%s) %d vs dst(%s) %d", c.src.Fs(), c.src.Size(), newDst.Fs(), newDst.Size())
+ }
+ // Verify hashes are the same after transfer - ignoring blank hashes
+ if c.hashType != hash.None {
+ // checkHashes has logs and counts errors
+ equal, _, srcSum, dstSum, _ := checkHashes(ctx, c.src, newDst, c.hashType)
+ if !equal {
+ return fmt.Errorf("corrupted on transfer: %v hashes differ src(%s) %q vs dst(%s) %q", c.hashType, c.src.Fs(), srcSum, newDst.Fs(), dstSum)
+ }
+ }
+ return nil
+}
+
+// copy src object to dst or f if nil. If dst is nil then it uses
+// remote as the name of the new object.
+//
+// It returns the destination object if possible. Note that this may
+// be nil.
+func (c *copy) copy(ctx context.Context) (newDst fs.Object, err error) {
+ var actionTaken string
+ retry := true
+ for tries := 0; retry && tries < c.maxTries; tries++ {
+ // Check we haven't hit any accounting limits
+ err = c.checkLimits(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ // Try server side copy
+ actionTaken, newDst, err = c.serverSideCopy(ctx)
+
+ // If can't server-side copy, do it manually
+ if errors.Is(err, fs.ErrorCantCopy) {
+ actionTaken, newDst, err = c.manualCopy(ctx)
+ }
+
+ // End if ctx is in error
+ if fserrors.ContextError(ctx, &err) {
+ break
+ }
+
+ // Retry if err returned a retry error
+ retry = false
+ if fserrors.IsRetryError(err) || fserrors.ShouldRetry(err) {
+ retry = true
+ } else if t, ok := pacer.IsRetryAfter(err); ok {
+ fs.Debugf(c.src, "Sleeping for %v (as indicated by the server) to obey Retry-After error: %v", t, err)
+ time.Sleep(t)
+ retry = true
+ }
+ if retry {
+ fs.Debugf(c.src, "Received error: %v - low level retry %d/%d", err, tries, c.maxTries)
+ c.tr.Reset(ctx) // skip incomplete accounting - will be overwritten by retry
+ continue
+ }
+ }
+ if err != nil {
+ err = fs.CountError(ctx, err)
+ fs.Errorf(c.src, "Failed to copy: %v", err)
+ if !c.inplace {
+ c.removeFailedPartialCopy(ctx, c.f, c.remoteForCopy)
+ }
+ return newDst, err
+ }
+
+ // Verify the copy
+ err = c.verify(ctx, newDst)
+ if err != nil {
+ fs.Errorf(newDst, "%v", err)
+ err = fs.CountError(ctx, err)
+ c.removeFailedCopy(ctx, newDst)
+ return nil, err
+ }
+
+ // Move the copied file to its real destination.
+ if !c.inplace && c.remoteForCopy != c.remote {
+ movedNewDst, err := c.dstFeatures.Move(ctx, newDst, c.remote)
+ if err != nil {
+ fs.Errorf(newDst, "partial file rename failed: %v", err)
+ err = fs.CountError(ctx, err)
+ c.removeFailedCopy(ctx, newDst)
+ return nil, err
+ }
+ fs.Debugf(newDst, "renamed to: %s", c.remote)
+ newDst = movedNewDst
+ }
+
+ // Log what we have done
+ if newDst != nil && c.src.String() != newDst.String() {
+ actionTaken = fmt.Sprintf("%s to: %s", actionTaken, newDst.String())
+ }
+ fs.Infof(c.src, "%s%s", actionTaken, fs.LogValueHide("size", fs.SizeSuffix(c.src.Size())))
+
+ return newDst, nil
+}
+
+// Copy src object to dst or f if nil. If dst is nil then it uses
+// remote as the name of the new object.
+//
+// It returns the destination object if possible. Note that this may
+// be nil.
+func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
+ ci := fs.GetConfig(ctx)
+ tr := accounting.Stats(ctx).NewTransfer(src, f)
+ defer func() {
+ tr.Done(ctx, err)
+ }()
+ if SkipDestructive(ctx, src, "copy") {
+ in := tr.Account(ctx, nil)
+ in.DryRun(src.Size())
+ return newDst, nil
+ }
+ c := &copy{
+ f: f,
+ dstFeatures: f.Features(),
+ dst: dst,
+ remote: transform.Path(ctx, remote, false),
+ src: src,
+ ci: ci,
+ tr: tr,
+ maxTries: ci.LowLevelRetries,
+ doUpdate: dst != nil,
+ }
+ c.hashType, c.hashOption = CommonHash(ctx, f, src.Fs())
+ if c.dst != nil {
+ c.remote = transform.Path(ctx, c.dst.Remote(), false)
+ }
+ // Are we using partials?
+ //
+ // If so set the flag and update the name we use for the copy
+ c.remoteForCopy, c.inplace, err = c.checkPartial(ctx)
+ if err != nil {
+ return nil, err
+ }
+ // Do the copy now everything is set up
+ return c.copy(ctx)
+}
+
+// CopyFile moves a single file possibly to a new name
+func CopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) {
+ return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, true, false)
+}
diff --git a/fs/operations/copy_test.go b/fs/operations/copy_test.go
new file mode 100644
index 0000000..e717f3b
--- /dev/null
+++ b/fs/operations/copy_test.go
@@ -0,0 +1,534 @@
+package operations_test
+
+import (
+ "context"
+ "crypto/rand"
+ "errors"
+ "fmt"
+ "os"
+ "path"
+ "runtime"
+ "sort"
+ "strings"
+ "testing"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/fs/sync"
+ "github.com/rclone/rclone/fstest"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestTruncateString(t *testing.T) {
+ for _, test := range []struct {
+ in string
+ n int
+ want string
+ }{
+ {
+ in: "",
+ n: 0,
+ want: "",
+ }, {
+ in: "Hello World",
+ n: 5,
+ want: "Hello",
+ }, {
+ in: "ááááá",
+ n: 5,
+ want: "áá",
+ }, {
+ in: "ááááá\xFF\xFF",
+ n: 5,
+ want: "áá\xc3",
+ }, {
+ in: "世世世世世",
+ n: 7,
+ want: "世世",
+ }, {
+ in: "🙂🙂🙂🙂🙂",
+ n: 16,
+ want: "🙂🙂🙂🙂",
+ }, {
+ in: "🙂🙂🙂🙂🙂",
+ n: 15,
+ want: "🙂🙂🙂",
+ }, {
+ in: "🙂🙂🙂🙂🙂",
+ n: 14,
+ want: "🙂🙂🙂",
+ }, {
+ in: "🙂🙂🙂🙂🙂",
+ n: 13,
+ want: "🙂🙂🙂",
+ }, {
+ in: "🙂🙂🙂🙂🙂",
+ n: 12,
+ want: "🙂🙂🙂",
+ }, {
+ in: "🙂🙂🙂🙂🙂",
+ n: 11,
+ want: "🙂🙂",
+ }, {
+ in: "𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
+ n: 100,
+ want: "𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢ",
+ }, {
+ in: "a𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
+ n: 100,
+ want: "a𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢ",
+ }, {
+ in: "aa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
+ n: 100,
+ want: "aa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱ",
+ }, {
+ in: "aaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
+ n: 100,
+ want: "aaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱ",
+ }, {
+ in: "aaaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ",
+ n: 100,
+ want: "aaaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽ",
+ },
+ } {
+ got := operations.TruncateString(test.in, test.n)
+ assert.Equal(t, test.want, got, fmt.Sprintf("In %q", test.in))
+ assert.LessOrEqual(t, len(got), test.n)
+ assert.GreaterOrEqual(t, len(got), test.n-3)
+ }
+}
+
+func TestCopyFile(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+
+ file1 := r.WriteFile("file1", "file1 contents", t1)
+ r.CheckLocalItems(t, file1)
+
+ file2 := file1
+ file2.Path = "sub/file2"
+
+ err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, file2)
+
+ err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, file2)
+
+ err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, file2)
+}
+
+// Find the longest file name for writing to local
+func maxLengthFileName(t *testing.T, r *fstest.Run) string {
+ require.NoError(t, r.Flocal.Mkdir(context.Background(), "")) // create the root
+ const maxLen = 16 * 1024
+ name := strings.Repeat("A", maxLen)
+ i := sort.Search(len(name), func(i int) (fail bool) {
+ filePath := path.Join(r.LocalName, name[:i])
+ err := os.WriteFile(filePath, []byte{0}, 0777)
+ if err != nil {
+ return true
+ }
+ err = os.Remove(filePath)
+ if err != nil {
+ t.Logf("Failed to remove test file: %v", err)
+ }
+ return false
+ })
+ return name[:i-1]
+}
+
+// Check we can copy a file of maximum name length
+func TestCopyLongFile(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ if !r.Fremote.Features().IsLocal {
+ t.Skip("Test only runs on local")
+ }
+
+ // Find the maximum length of file we can write
+ name := maxLengthFileName(t, r)
+ t.Logf("Max length of file name is %d", len(name))
+ file1 := r.WriteFile(name, "file1 contents", t1)
+ r.CheckLocalItems(t, file1)
+
+ err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, file1)
+}
+
+func TestCopyFileBackupDir(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+ if !operations.CanServerSideMove(r.Fremote) {
+ t.Skip("Skipping test as remote does not support server-side move or copy")
+ }
+
+ ci.BackupDir = r.FremoteName + "/backup"
+
+ file1 := r.WriteFile("dst/file1", "file1 contents", t1)
+ r.CheckLocalItems(t, file1)
+
+ file1old := r.WriteObject(ctx, "dst/file1", "file1 contents old", t1)
+ r.CheckRemoteItems(t, file1old)
+
+ err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, file1)
+ file1old.Path = "backup/dst/file1"
+ r.CheckRemoteItems(t, file1old, file1)
+}
+
+// Test with CompareDest set
+func TestCopyFileCompareDest(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+
+ ci.CompareDest = []string{r.FremoteName + "/CompareDest"}
+ fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
+ require.NoError(t, err)
+
+ // check empty dest, empty compare
+ file1 := r.WriteFile("one", "one", t1)
+ r.CheckLocalItems(t, file1)
+
+ err = operations.CopyFile(ctx, fdst, r.Flocal, file1.Path, file1.Path)
+ require.NoError(t, err)
+
+ file1dst := file1
+ file1dst.Path = "dst/one"
+
+ r.CheckRemoteItems(t, file1dst)
+
+ // check old dest, empty compare
+ file1b := r.WriteFile("one", "onet2", t2)
+ r.CheckRemoteItems(t, file1dst)
+ r.CheckLocalItems(t, file1b)
+
+ err = operations.CopyFile(ctx, fdst, r.Flocal, file1b.Path, file1b.Path)
+ require.NoError(t, err)
+
+ file1bdst := file1b
+ file1bdst.Path = "dst/one"
+
+ r.CheckRemoteItems(t, file1bdst)
+
+ // check old dest, new compare
+ file3 := r.WriteObject(ctx, "dst/one", "one", t1)
+ file2 := r.WriteObject(ctx, "CompareDest/one", "onet2", t2)
+ file1c := r.WriteFile("one", "onet2", t2)
+ r.CheckRemoteItems(t, file2, file3)
+ r.CheckLocalItems(t, file1c)
+
+ err = operations.CopyFile(ctx, fdst, r.Flocal, file1c.Path, file1c.Path)
+ require.NoError(t, err)
+
+ r.CheckRemoteItems(t, file2, file3)
+
+ // check empty dest, new compare
+ file4 := r.WriteObject(ctx, "CompareDest/two", "two", t2)
+ file5 := r.WriteFile("two", "two", t2)
+ r.CheckRemoteItems(t, file2, file3, file4)
+ r.CheckLocalItems(t, file1c, file5)
+
+ err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
+ require.NoError(t, err)
+
+ r.CheckRemoteItems(t, file2, file3, file4)
+
+ // check new dest, new compare
+ err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
+ require.NoError(t, err)
+
+ r.CheckRemoteItems(t, file2, file3, file4)
+
+ // check empty dest, old compare
+ file5b := r.WriteFile("two", "twot3", t3)
+ r.CheckRemoteItems(t, file2, file3, file4)
+ r.CheckLocalItems(t, file1c, file5b)
+
+ err = operations.CopyFile(ctx, fdst, r.Flocal, file5b.Path, file5b.Path)
+ require.NoError(t, err)
+
+ file5bdst := file5b
+ file5bdst.Path = "dst/two"
+
+ r.CheckRemoteItems(t, file2, file3, file4, file5bdst)
+}
+
+// Test with CopyDest set
+func TestCopyFileCopyDest(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+
+ if r.Fremote.Features().Copy == nil {
+ t.Skip("Skipping test as remote does not support server-side copy")
+ }
+
+ ci.CopyDest = []string{r.FremoteName + "/CopyDest"}
+
+ fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
+ require.NoError(t, err)
+
+ // check empty dest, empty copy
+ file1 := r.WriteFile("one", "one", t1)
+ r.CheckLocalItems(t, file1)
+
+ err = operations.CopyFile(ctx, fdst, r.Flocal, file1.Path, file1.Path)
+ require.NoError(t, err)
+
+ file1dst := file1
+ file1dst.Path = "dst/one"
+
+ r.CheckRemoteItems(t, file1dst)
+
+ // check old dest, empty copy
+ file1b := r.WriteFile("one", "onet2", t2)
+ r.CheckRemoteItems(t, file1dst)
+ r.CheckLocalItems(t, file1b)
+
+ err = operations.CopyFile(ctx, fdst, r.Flocal, file1b.Path, file1b.Path)
+ require.NoError(t, err)
+
+ file1bdst := file1b
+ file1bdst.Path = "dst/one"
+
+ r.CheckRemoteItems(t, file1bdst)
+
+ // check old dest, new copy, backup-dir
+
+ ci.BackupDir = r.FremoteName + "/BackupDir"
+
+ file3 := r.WriteObject(ctx, "dst/one", "one", t1)
+ file2 := r.WriteObject(ctx, "CopyDest/one", "onet2", t2)
+ file1c := r.WriteFile("one", "onet2", t2)
+ r.CheckRemoteItems(t, file2, file3)
+ r.CheckLocalItems(t, file1c)
+
+ err = operations.CopyFile(ctx, fdst, r.Flocal, file1c.Path, file1c.Path)
+ require.NoError(t, err)
+
+ file2dst := file2
+ file2dst.Path = "dst/one"
+ file3.Path = "BackupDir/one"
+
+ r.CheckRemoteItems(t, file2, file2dst, file3)
+ ci.BackupDir = ""
+
+ // check empty dest, new copy
+ file4 := r.WriteObject(ctx, "CopyDest/two", "two", t2)
+ file5 := r.WriteFile("two", "two", t2)
+ r.CheckRemoteItems(t, file2, file2dst, file3, file4)
+ r.CheckLocalItems(t, file1c, file5)
+
+ err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
+ require.NoError(t, err)
+
+ file4dst := file4
+ file4dst.Path = "dst/two"
+
+ r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst)
+
+ // check new dest, new copy
+ err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path)
+ require.NoError(t, err)
+
+ r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst)
+
+ // check empty dest, old copy
+ file6 := r.WriteObject(ctx, "CopyDest/three", "three", t2)
+ file7 := r.WriteFile("three", "threet3", t3)
+ r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6)
+ r.CheckLocalItems(t, file1c, file5, file7)
+
+ err = operations.CopyFile(ctx, fdst, r.Flocal, file7.Path, file7.Path)
+ require.NoError(t, err)
+
+ file7dst := file7
+ file7dst.Path = "dst/three"
+
+ r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6, file7dst)
+}
+
+func TestCopyInplace(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+
+ if !r.Fremote.Features().PartialUploads {
+ t.Skip("Partial uploads not supported")
+ }
+
+ ci.Inplace = true
+
+ file1 := r.WriteFile("file1", "file1 contents", t1)
+ r.CheckLocalItems(t, file1)
+
+ file2 := file1
+ file2.Path = "sub/file2"
+
+ err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, file2)
+
+ err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, file2)
+
+ err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, file2)
+}
+
+func TestCopyLongFileName(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+
+ if !r.Fremote.Features().PartialUploads {
+ t.Skip("Partial uploads not supported")
+ }
+
+ ci.Inplace = false // the default
+
+ file1 := r.WriteFile("file1", "file1 contents", t1)
+ r.CheckLocalItems(t, file1)
+
+ file2 := file1
+ file2.Path = "sub/" + strings.Repeat("file2", 30)
+
+ err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, file2)
+
+ err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, file2)
+
+ err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, file2)
+}
+
+func TestCopyLongFileNameCollision(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+
+ if !r.Fremote.Features().PartialUploads {
+ t.Skip("Partial uploads not supported")
+ }
+
+ ci.Inplace = false
+ ci.Transfers = 4
+
+ // Write a lot of identical files with long names
+ files := make([]fstest.Item, 10)
+ namePrefix := strings.Repeat("file1", 30)
+ for i := range files {
+ files[i] = r.WriteFile(fmt.Sprintf("%s%02d", namePrefix, i), "file1 contents", t1)
+ }
+ r.CheckLocalItems(t, files...)
+
+ err := sync.CopyDir(ctx, r.Fremote, r.Flocal, false)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, files...)
+ r.CheckRemoteItems(t, files...)
+}
+
+func TestCopyFileMaxTransfer(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+ defer accounting.Stats(ctx).ResetCounters()
+
+ const sizeCutoff = 2048
+
+ // Make random incompressible data
+ randomData := make([]byte, sizeCutoff)
+ _, err := rand.Read(randomData)
+ require.NoError(t, err)
+ randomString := string(randomData)
+
+ file1 := r.WriteFile("TestCopyFileMaxTransfer/file1", "file1 contents", t1)
+ file2 := r.WriteFile("TestCopyFileMaxTransfer/file2", "file2 contents"+randomString, t2)
+ file3 := r.WriteFile("TestCopyFileMaxTransfer/file3", "file3 contents"+randomString, t2)
+ file4 := r.WriteFile("TestCopyFileMaxTransfer/file4", "file4 contents"+randomString, t2)
+
+ // Cutoff mode: Hard
+ ci.MaxTransfer = sizeCutoff
+ ci.CutoffMode = fs.CutoffModeHard
+
+ if runtime.GOOS == "darwin" {
+ // disable server-side copies as they don't count towards transfer size stats
+ r.Flocal.Features().Disable("Copy")
+ if r.Fremote.Features().IsLocal {
+ r.Fremote.Features().Disable("Copy")
+ }
+ }
+
+ // file1: Show a small file gets transferred OK
+ accounting.Stats(ctx).ResetCounters()
+ err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, file1, file2, file3, file4)
+ r.CheckRemoteItems(t, file1)
+
+ // file2: show a large file does not get transferred
+ accounting.Stats(ctx).ResetCounters()
+ err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file2.Path)
+ require.NotNil(t, err, "Did not get expected max transfer limit error")
+ if !errors.Is(err, accounting.ErrorMaxTransferLimitReachedFatal) {
+ t.Log("Expecting error to contain accounting.ErrorMaxTransferLimitReachedFatal")
+ // Sometimes the backends or their SDKs don't pass the
+ // error through properly, so check that it at least
+ // has the text we expect in.
+ assert.Contains(t, err.Error(), "max transfer limit reached")
+ }
+ r.CheckLocalItems(t, file1, file2, file3, file4)
+ r.CheckRemoteItems(t, file1)
+
+ // Cutoff mode: Cautious
+ ci.CutoffMode = fs.CutoffModeCautious
+
+ // file3: show a large file does not get transferred
+ accounting.Stats(ctx).ResetCounters()
+ err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file3.Path, file3.Path)
+ require.NotNil(t, err)
+ assert.True(t, errors.Is(err, accounting.ErrorMaxTransferLimitReachedGraceful))
+ r.CheckLocalItems(t, file1, file2, file3, file4)
+ r.CheckRemoteItems(t, file1)
+
+ if isChunker(r.Fremote) {
+ t.Log("skipping remainder of test for chunker as it involves multiple transfers")
+ return
+ }
+
+ // Cutoff mode: Soft
+ ci.CutoffMode = fs.CutoffModeSoft
+
+ // file4: show a large file does get transferred this time
+ accounting.Stats(ctx).ResetCounters()
+ err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file4.Path, file4.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, file1, file2, file3, file4)
+ r.CheckRemoteItems(t, file1, file4)
+}
diff --git a/fs/operations/dedupe.go b/fs/operations/dedupe.go
new file mode 100644
index 0000000..60c66c4
--- /dev/null
+++ b/fs/operations/dedupe.go
@@ -0,0 +1,506 @@
+// dedupe - gets rid of identical files remotes which can have duplicate file names (drive, mega)
+
+package operations
+
+import (
+ "context"
+ "fmt"
+ "path"
+ "sort"
+ "strings"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/config"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/walk"
+)
+
+// dedupeRename renames the objs slice to different names
+func dedupeRename(ctx context.Context, f fs.Fs, remote string, objs []fs.Object) {
+ doMove := f.Features().Move
+ if doMove == nil {
+ fs.Fatalf(nil, "Fs %v doesn't support Move", f)
+ }
+ ext := path.Ext(remote)
+ base := remote[:len(remote)-len(ext)]
+
+outer:
+ for i, o := range objs {
+ suffix := 1
+ newName := fmt.Sprintf("%s-%d%s", base, i+suffix, ext)
+ _, err := f.NewObject(ctx, newName)
+ for ; err != fs.ErrorObjectNotFound; suffix++ {
+ if err != nil {
+ err = fs.CountError(ctx, err)
+ fs.Errorf(o, "Failed to check for existing object: %v", err)
+ continue outer
+ }
+ if suffix > 100 {
+ fs.Errorf(o, "Could not find an available new name")
+ continue outer
+ }
+ newName = fmt.Sprintf("%s-%d%s", base, i+suffix, ext)
+ _, err = f.NewObject(ctx, newName)
+ }
+ if !SkipDestructive(ctx, o, "rename") {
+ newObj, err := doMove(ctx, o, newName)
+ if err != nil {
+ err = fs.CountError(ctx, err)
+ fs.Errorf(o, "Failed to rename: %v", err)
+ continue
+ }
+ fs.Infof(newObj, "renamed from: %v", o)
+ }
+ }
+}
+
+// dedupeDeleteAllButOne deletes all but the one in keep
+func dedupeDeleteAllButOne(ctx context.Context, keep int, remote string, objs []fs.Object) {
+ count := 0
+ for i, o := range objs {
+ if i == keep {
+ continue
+ }
+ err := DeleteFile(ctx, o)
+ if err == nil {
+ count++
+ }
+ }
+ if count > 0 {
+ fs.Logf(remote, "Deleted %d extra copies", count)
+ }
+}
+
+// dedupeDeleteIdentical deletes all but one of identical (by hash) copies
+func dedupeDeleteIdentical(ctx context.Context, ht hash.Type, remote string, objs []fs.Object) (remainingObjs []fs.Object) {
+ ci := fs.GetConfig(ctx)
+
+ // Make map of IDs
+ IDs := make(map[string]int, len(objs))
+ for _, o := range objs {
+ if do, ok := o.(fs.IDer); ok {
+ if ID := do.ID(); ID != "" {
+ IDs[ID]++
+ }
+ }
+ }
+
+ // Remove duplicate IDs
+ newObjs := objs[:0]
+ for _, o := range objs {
+ if do, ok := o.(fs.IDer); ok {
+ if ID := do.ID(); ID != "" {
+ if IDs[ID] <= 1 {
+ newObjs = append(newObjs, o)
+ } else {
+ fs.Logf(o, "Ignoring as it appears %d times in the listing and deleting would lead to data loss", IDs[ID])
+ }
+ }
+ }
+ }
+ objs = newObjs
+
+ // See how many of these duplicates are identical
+ dupesByID := make(map[string][]fs.Object, len(objs))
+ for _, o := range objs {
+ ID := ""
+ if ci.SizeOnly && o.Size() >= 0 {
+ ID = fmt.Sprintf("size %d", o.Size())
+ } else if ht != hash.None {
+ hashValue, err := o.Hash(ctx, ht)
+ if err == nil && hashValue != "" {
+ ID = fmt.Sprintf("%v %s", ht, hashValue)
+ }
+ }
+ if ID == "" {
+ remainingObjs = append(remainingObjs, o)
+ } else {
+ dupesByID[ID] = append(dupesByID[ID], o)
+ }
+ }
+
+ // Delete identical duplicates, filling remainingObjs with the ones remaining
+ for ID, dupes := range dupesByID {
+ remainingObjs = append(remainingObjs, dupes[0])
+ if len(dupes) > 1 {
+ fs.Logf(remote, "Deleting %d/%d identical duplicates (%s)", len(dupes)-1, len(dupes), ID)
+ for _, o := range dupes[1:] {
+ err := DeleteFile(ctx, o)
+ if err != nil {
+ remainingObjs = append(remainingObjs, o)
+ }
+ }
+ }
+ }
+
+ return remainingObjs
+}
+
+// dedupeList lists the duplicates and does nothing
+func dedupeList(ctx context.Context, f fs.Fs, ht hash.Type, remote string, objs []fs.Object, byHash bool) {
+ fmt.Printf("%s: %d duplicates\n", remote, len(objs))
+ for i, o := range objs {
+ hashValue := ""
+ if ht != hash.None {
+ var err error
+ hashValue, err = o.Hash(ctx, ht)
+ if err != nil {
+ hashValue = err.Error()
+ }
+ }
+ if byHash {
+ fmt.Printf(" %d: %12d bytes, %s, %s\n", i+1, o.Size(), o.ModTime(ctx).Local().Format("2006-01-02 15:04:05.000000000"), o.Remote())
+ } else {
+ fmt.Printf(" %d: %12d bytes, %s, %v %32s\n", i+1, o.Size(), o.ModTime(ctx).Local().Format("2006-01-02 15:04:05.000000000"), ht, hashValue)
+ }
+ }
+}
+
+// dedupeInteractive interactively dedupes the slice of objects
+func dedupeInteractive(ctx context.Context, f fs.Fs, ht hash.Type, remote string, objs []fs.Object, byHash bool) bool {
+ dedupeList(ctx, f, ht, remote, objs, byHash)
+ commands := []string{"sSkip and do nothing", "kKeep just one (choose which in next step)"}
+ if !byHash {
+ commands = append(commands, "rRename all to be different (by changing file.jpg to file-1.jpg)")
+ }
+ commands = append(commands, "qQuit")
+ switch config.Command(commands) {
+ case 's':
+ case 'k':
+ keep := config.ChooseNumber("Enter the number of the file to keep", 1, len(objs))
+ dedupeDeleteAllButOne(ctx, keep-1, remote, objs)
+ case 'r':
+ dedupeRename(ctx, f, remote, objs)
+ case 'q':
+ return false
+ }
+ return true
+}
+
+// DeduplicateMode is how the dedupe command chooses what to do
+type DeduplicateMode int
+
+// Deduplicate modes
+const (
+ DeduplicateInteractive DeduplicateMode = iota // interactively ask the user
+ DeduplicateSkip // skip all conflicts
+ DeduplicateFirst // choose the first object
+ DeduplicateNewest // choose the newest object
+ DeduplicateOldest // choose the oldest object
+ DeduplicateRename // rename the objects
+ DeduplicateLargest // choose the largest object
+ DeduplicateSmallest // choose the smallest object
+ DeduplicateList // list duplicates only
+)
+
+func (x DeduplicateMode) String() string {
+ switch x {
+ case DeduplicateInteractive:
+ return "interactive"
+ case DeduplicateSkip:
+ return "skip"
+ case DeduplicateFirst:
+ return "first"
+ case DeduplicateNewest:
+ return "newest"
+ case DeduplicateOldest:
+ return "oldest"
+ case DeduplicateRename:
+ return "rename"
+ case DeduplicateLargest:
+ return "largest"
+ case DeduplicateSmallest:
+ return "smallest"
+ case DeduplicateList:
+ return "list"
+ }
+ return "unknown"
+}
+
+// Set a DeduplicateMode from a string
+func (x *DeduplicateMode) Set(s string) error {
+ switch strings.ToLower(s) {
+ case "interactive":
+ *x = DeduplicateInteractive
+ case "skip":
+ *x = DeduplicateSkip
+ case "first":
+ *x = DeduplicateFirst
+ case "newest":
+ *x = DeduplicateNewest
+ case "oldest":
+ *x = DeduplicateOldest
+ case "rename":
+ *x = DeduplicateRename
+ case "largest":
+ *x = DeduplicateLargest
+ case "smallest":
+ *x = DeduplicateSmallest
+ case "list":
+ *x = DeduplicateList
+ default:
+ return fmt.Errorf("unknown mode for dedupe %q", s)
+ }
+ return nil
+}
+
+// Type of the value
+func (x *DeduplicateMode) Type() string {
+ return "string"
+}
+
+// Directory with entry count and links to parents
+type dedupeDir struct {
+ dir fs.Directory
+ parent string
+ count int
+}
+
+// Map of directories by ID with recursive counts
+type dedupeDirsMap map[string]*dedupeDir
+
+func (dm dedupeDirsMap) get(id string) *dedupeDir {
+ d := dm[id]
+ if d == nil {
+ d = &dedupeDir{}
+ dm[id] = d
+ }
+ return d
+}
+
+func (dm dedupeDirsMap) increment(parent string) {
+ if parent != "" {
+ d := dm.get(parent)
+ d.count++
+ dm.increment(d.parent)
+ }
+}
+
+// dedupeFindDuplicateDirs scans f for duplicate directories
+func dedupeFindDuplicateDirs(ctx context.Context, f fs.Fs) (duplicateDirs [][]*dedupeDir, err error) {
+ dirsByID := dedupeDirsMap{}
+ dirs := map[string][]*dedupeDir{}
+
+ ci := fs.GetConfig(ctx)
+ err = walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListAll, func(entries fs.DirEntries) error {
+ for _, entry := range entries {
+ tr := accounting.Stats(ctx).NewCheckingTransfer(entry, "merging")
+
+ remote := entry.Remote()
+ parentRemote := path.Dir(remote)
+ if parentRemote == "." {
+ parentRemote = ""
+ }
+
+ // Obtain ID of the object parent, if known.
+ // (This usually means that backend allows duplicate paths)
+ // Fall back to remote parent path, if unavailable.
+ var parent string
+ if entryParentIDer, ok := entry.(fs.ParentIDer); ok {
+ parent = entryParentIDer.ParentID()
+ }
+ if parent == "" {
+ parent = parentRemote
+ }
+
+ var ID string
+ if entryIDer, ok := entry.(fs.IDer); ok {
+ ID = entryIDer.ID()
+ }
+ if ID == "" {
+ ID = remote
+ }
+
+ if fsDir, ok := entry.(fs.Directory); ok {
+ d := dirsByID.get(ID)
+ d.dir = fsDir
+ d.parent = parent
+ dirs[remote] = append(dirs[remote], d)
+ }
+
+ dirsByID.increment(parent)
+ tr.Done(ctx, nil)
+ }
+ return nil
+ })
+ if err != nil {
+ return nil, fmt.Errorf("find duplicate dirs: %w", err)
+ }
+
+ // Make sure parents are before children
+ duplicateNames := []string{}
+ for name, ds := range dirs {
+ if len(ds) > 1 {
+ duplicateNames = append(duplicateNames, name)
+ }
+ }
+ sort.Strings(duplicateNames)
+ for _, name := range duplicateNames {
+ duplicateDirs = append(duplicateDirs, dirs[name])
+ }
+
+ return
+}
+
+// dedupeMergeDuplicateDirs merges all the duplicate directories found
+func dedupeMergeDuplicateDirs(ctx context.Context, f fs.Fs, duplicateDirs [][]*dedupeDir) error {
+ mergeDirs := f.Features().MergeDirs
+ if mergeDirs == nil {
+ return fmt.Errorf("%v: can't merge directories", f)
+ }
+ dirCacheFlush := f.Features().DirCacheFlush
+ if dirCacheFlush == nil {
+ return fmt.Errorf("%v: can't flush dir cache", f)
+ }
+ for _, dedupeDirs := range duplicateDirs {
+ if SkipDestructive(ctx, dedupeDirs[0].dir, "merge duplicate directories") {
+ continue
+ }
+
+ // Put largest directory in front to minimize movements
+ fsDirs := []fs.Directory{}
+ largestCount := -1
+ largestIdx := 0
+ for i, d := range dedupeDirs {
+ fsDirs = append(fsDirs, d.dir)
+ if d.count > largestCount {
+ largestIdx = i
+ largestCount = d.count
+ }
+ }
+ fsDirs[largestIdx], fsDirs[0] = fsDirs[0], fsDirs[largestIdx]
+
+ fs.Infof(fsDirs[0], "Merging contents of duplicate directories")
+ err := mergeDirs(ctx, fsDirs)
+ if err != nil {
+ err = fs.CountError(ctx, err)
+ fs.Errorf(nil, "merge duplicate dirs: %v", err)
+ }
+ }
+ dirCacheFlush()
+ return nil
+}
+
+// sort oldest first
+func sortOldestFirst(objs []fs.Object) {
+ sort.Slice(objs, func(i, j int) bool {
+ return objs[i].ModTime(context.TODO()).Before(objs[j].ModTime(context.TODO()))
+ })
+}
+
+// sort smallest first
+func sortSmallestFirst(objs []fs.Object) {
+ sort.Slice(objs, func(i, j int) bool {
+ return objs[i].Size() < objs[j].Size()
+ })
+}
+
+// Deduplicate interactively finds duplicate files and offers to
+// delete all but one or rename them to be different. Only useful with
+// Google Drive which can have duplicate file names.
+func Deduplicate(ctx context.Context, f fs.Fs, mode DeduplicateMode, byHash bool) error {
+ ci := fs.GetConfig(ctx)
+ // find a hash to use
+ ht := f.Hashes().GetOne()
+ what := "names"
+ if byHash {
+ if ht == hash.None {
+ return fmt.Errorf("%v has no hashes", f)
+ }
+ what = ht.String() + " hashes"
+ }
+ fs.Infof(f, "Looking for duplicate %s using %v mode.", what, mode)
+
+ // Find duplicate directories first and fix them
+ if !byHash {
+ duplicateDirs, err := dedupeFindDuplicateDirs(ctx, f)
+ if err != nil {
+ return err
+ }
+ if len(duplicateDirs) > 0 {
+ if mode != DeduplicateList {
+ err = dedupeMergeDuplicateDirs(ctx, f, duplicateDirs)
+ if err != nil {
+ return err
+ }
+ } else {
+ for _, dedupeDirs := range duplicateDirs {
+ remote := dedupeDirs[0].dir.Remote()
+ fmt.Printf("%s: %d duplicates of this directory\n", remote, len(dedupeDirs))
+ }
+ }
+ }
+ }
+
+ // Now find duplicate files
+ files := map[string][]fs.Object{}
+ err := walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
+ entries.ForObject(func(o fs.Object) {
+ tr := accounting.Stats(ctx).NewCheckingTransfer(o, "checking")
+ defer tr.Done(ctx, nil)
+
+ var remote string
+ var err error
+ if byHash {
+ remote, err = o.Hash(ctx, ht)
+ if err != nil {
+ fs.Errorf(o, "Failed to hash: %v", err)
+ remote = ""
+ }
+ } else {
+ remote = o.Remote()
+ }
+ if remote != "" {
+ files[remote] = append(files[remote], o)
+ }
+ })
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+
+ for remote, objs := range files {
+ if len(objs) <= 1 {
+ continue
+ }
+ fs.Logf(remote, "Found %d files with duplicate %s", len(objs), what)
+ if !byHash && mode != DeduplicateList {
+ objs = dedupeDeleteIdentical(ctx, ht, remote, objs)
+ if len(objs) <= 1 {
+ fs.Logf(remote, "All duplicates removed")
+ continue
+ }
+ }
+ switch mode {
+ case DeduplicateInteractive:
+ if !dedupeInteractive(ctx, f, ht, remote, objs, byHash) {
+ return nil
+ }
+ case DeduplicateFirst:
+ dedupeDeleteAllButOne(ctx, 0, remote, objs)
+ case DeduplicateNewest:
+ sortOldestFirst(objs)
+ dedupeDeleteAllButOne(ctx, len(objs)-1, remote, objs)
+ case DeduplicateOldest:
+ sortOldestFirst(objs)
+ dedupeDeleteAllButOne(ctx, 0, remote, objs)
+ case DeduplicateRename:
+ dedupeRename(ctx, f, remote, objs)
+ case DeduplicateLargest:
+ sortSmallestFirst(objs)
+ dedupeDeleteAllButOne(ctx, len(objs)-1, remote, objs)
+ case DeduplicateSmallest:
+ sortSmallestFirst(objs)
+ dedupeDeleteAllButOne(ctx, 0, remote, objs)
+ case DeduplicateSkip:
+ fs.Logf(remote, "Skipping %d files with duplicate %s", len(objs), what)
+ case DeduplicateList:
+ dedupeList(ctx, f, ht, remote, objs, byHash)
+ default:
+ //skip
+ }
+ }
+ return nil
+}
diff --git a/fs/operations/dedupe_test.go b/fs/operations/dedupe_test.go
new file mode 100644
index 0000000..15a4e2d
--- /dev/null
+++ b/fs/operations/dedupe_test.go
@@ -0,0 +1,280 @@
+package operations_test
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/fstest"
+ "github.com/rclone/rclone/lib/random"
+ "github.com/spf13/pflag"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// Check flag satisfies the interface
+var _ pflag.Value = (*operations.DeduplicateMode)(nil)
+
+func skipIfCantDedupe(t *testing.T, f fs.Fs) {
+ if !f.Features().DuplicateFiles {
+ t.Skip("Can't test deduplicate - no duplicate files possible")
+ }
+ if f.Features().PutUnchecked == nil {
+ t.Skip("Can't test deduplicate - no PutUnchecked")
+ }
+ if f.Features().MergeDirs == nil {
+ t.Skip("Can't test deduplicate - no MergeDirs")
+ }
+}
+
+func skipIfNoHash(t *testing.T, f fs.Fs) {
+ if f.Hashes().GetOne() == hash.None {
+ t.Skip("Can't run this test without a hash")
+ }
+}
+
+func skipIfNoModTime(t *testing.T, f fs.Fs) {
+ if f.Precision() >= fs.ModTimeNotSupported {
+ t.Skip("Can't run this test without modtimes")
+ }
+}
+
+func TestDeduplicateInteractive(t *testing.T) {
+ r := fstest.NewRun(t)
+ skipIfCantDedupe(t, r.Fremote)
+ skipIfNoHash(t, r.Fremote)
+
+ file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
+ file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
+ file3 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
+ r.CheckWithDuplicates(t, file1, file2, file3)
+
+ err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateInteractive, false)
+ require.NoError(t, err)
+
+ r.CheckRemoteItems(t, file1)
+}
+
+func TestDeduplicateSkip(t *testing.T) {
+ r := fstest.NewRun(t)
+ skipIfCantDedupe(t, r.Fremote)
+ haveHash := r.Fremote.Hashes().GetOne() != hash.None
+
+ file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
+ files := []fstest.Item{file1}
+ if haveHash {
+ file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
+ files = append(files, file2)
+ }
+ file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t1)
+ files = append(files, file3)
+ r.CheckWithDuplicates(t, files...)
+
+ err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateSkip, false)
+ require.NoError(t, err)
+
+ r.CheckWithDuplicates(t, file1, file3)
+}
+
+func TestDeduplicateSizeOnly(t *testing.T) {
+ r := fstest.NewRun(t)
+ skipIfCantDedupe(t, r.Fremote)
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
+
+ file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
+ file2 := r.WriteUncheckedObject(context.Background(), "one", "THIS IS ONE", t1)
+ file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t1)
+ r.CheckWithDuplicates(t, file1, file2, file3)
+
+ ci.SizeOnly = true
+ defer func() {
+ ci.SizeOnly = false
+ }()
+
+ err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateSkip, false)
+ require.NoError(t, err)
+
+ r.CheckWithDuplicates(t, file1, file3)
+}
+
+func TestDeduplicateFirst(t *testing.T) {
+ r := fstest.NewRun(t)
+ skipIfCantDedupe(t, r.Fremote)
+
+ file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
+ file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one A", t1)
+ file3 := r.WriteUncheckedObject(context.Background(), "one", "This is one BB", t1)
+ r.CheckWithDuplicates(t, file1, file2, file3)
+
+ err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateFirst, false)
+ require.NoError(t, err)
+
+ // list until we get one object
+ var objects, size int64
+ for try := 1; try <= *fstest.ListRetries; try++ {
+ objects, size, _, err = operations.Count(context.Background(), r.Fremote)
+ require.NoError(t, err)
+ if objects == 1 {
+ break
+ }
+ time.Sleep(time.Second)
+ }
+ assert.Equal(t, int64(1), objects)
+ if size != file1.Size && size != file2.Size && size != file3.Size {
+ t.Errorf("Size not one of the object sizes %d", size)
+ }
+}
+
+func TestDeduplicateNewest(t *testing.T) {
+ r := fstest.NewRun(t)
+ skipIfCantDedupe(t, r.Fremote)
+ skipIfNoModTime(t, r.Fremote)
+
+ file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
+ file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
+ file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
+ r.CheckWithDuplicates(t, file1, file2, file3)
+
+ err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateNewest, false)
+ require.NoError(t, err)
+
+ r.CheckRemoteItems(t, file3)
+}
+
+func TestDeduplicateNewestByHash(t *testing.T) {
+ r := fstest.NewRun(t)
+ skipIfNoHash(t, r.Fremote)
+ skipIfNoModTime(t, r.Fremote)
+ contents := random.String(100)
+
+ file1 := r.WriteObject(context.Background(), "one", contents, t1)
+ file2 := r.WriteObject(context.Background(), "also/one", contents, t2)
+ file3 := r.WriteObject(context.Background(), "another", contents, t3)
+ file4 := r.WriteObject(context.Background(), "not-one", "stuff", t3)
+ r.CheckRemoteItems(t, file1, file2, file3, file4)
+
+ err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateNewest, true)
+ require.NoError(t, err)
+
+ r.CheckRemoteItems(t, file3, file4)
+}
+
+func TestDeduplicateOldest(t *testing.T) {
+ r := fstest.NewRun(t)
+ skipIfCantDedupe(t, r.Fremote)
+
+ file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
+ file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
+ file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
+ r.CheckWithDuplicates(t, file1, file2, file3)
+
+ err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateOldest, false)
+ require.NoError(t, err)
+
+ r.CheckRemoteItems(t, file1)
+}
+
+func TestDeduplicateLargest(t *testing.T) {
+ r := fstest.NewRun(t)
+ skipIfCantDedupe(t, r.Fremote)
+
+ file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
+ file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
+ file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
+ r.CheckWithDuplicates(t, file1, file2, file3)
+
+ err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateLargest, false)
+ require.NoError(t, err)
+
+ r.CheckRemoteItems(t, file3)
+}
+
+func TestDeduplicateSmallest(t *testing.T) {
+ r := fstest.NewRun(t)
+ skipIfCantDedupe(t, r.Fremote)
+
+ file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
+ file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2)
+ file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3)
+ r.CheckWithDuplicates(t, file1, file2, file3)
+
+ err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateSmallest, false)
+ require.NoError(t, err)
+
+ r.CheckRemoteItems(t, file1)
+}
+
+func TestDeduplicateRename(t *testing.T) {
+ r := fstest.NewRun(t)
+ skipIfCantDedupe(t, r.Fremote)
+
+ file1 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is one", t1)
+ file2 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is one too", t2)
+ file3 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is another one", t3)
+ file4 := r.WriteUncheckedObject(context.Background(), "one-1.txt", "This is not a duplicate", t1)
+ r.CheckWithDuplicates(t, file1, file2, file3, file4)
+
+ err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateRename, false)
+ require.NoError(t, err)
+
+ require.NoError(t, walk.ListR(context.Background(), r.Fremote, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+ entries.ForObject(func(o fs.Object) {
+ remote := o.Remote()
+ if remote != "one-1.txt" &&
+ remote != "one-2.txt" &&
+ remote != "one-3.txt" &&
+ remote != "one-4.txt" {
+ t.Errorf("Bad file name after rename %q", remote)
+ }
+ size := o.Size()
+ if size != file1.Size &&
+ size != file2.Size &&
+ size != file3.Size &&
+ size != file4.Size {
+ t.Errorf("Size not one of the object sizes %d", size)
+ }
+ if remote == "one-1.txt" && size != file4.Size {
+ t.Errorf("Existing non-duplicate file modified %q", remote)
+ }
+ })
+ return nil
+ }))
+}
+
+// This should really be a unit test, but the test framework there
+// doesn't have enough tools to make it easy
+func TestMergeDirs(t *testing.T) {
+ r := fstest.NewRun(t)
+
+ mergeDirs := r.Fremote.Features().MergeDirs
+ if mergeDirs == nil {
+ t.Skip("Can't merge directories")
+ }
+
+ file1 := r.WriteObject(context.Background(), "dupe1/one.txt", "This is one", t1)
+ file2 := r.WriteObject(context.Background(), "dupe2/two.txt", "This is one too", t2)
+ file3 := r.WriteObject(context.Background(), "dupe3/three.txt", "This is another one", t3)
+
+ objs, dirs, err := walk.GetAll(context.Background(), r.Fremote, "", true, 1)
+ require.NoError(t, err)
+ assert.Equal(t, 3, len(dirs))
+ assert.Equal(t, 0, len(objs))
+
+ err = mergeDirs(context.Background(), dirs)
+ require.NoError(t, err)
+
+ file2.Path = "dupe1/two.txt"
+ file3.Path = "dupe1/three.txt"
+ r.CheckRemoteItems(t, file1, file2, file3)
+
+ objs, dirs, err = walk.GetAll(context.Background(), r.Fremote, "", true, 1)
+ require.NoError(t, err)
+ assert.Equal(t, 1, len(dirs))
+ assert.Equal(t, 0, len(objs))
+ assert.Equal(t, "dupe1", dirs[0].Remote())
+}
diff --git a/fs/operations/listdirsorted_test.go b/fs/operations/listdirsorted_test.go
new file mode 100644
index 0000000..4719d6c
--- /dev/null
+++ b/fs/operations/listdirsorted_test.go
@@ -0,0 +1,126 @@
+package operations_test
+
+import (
+ "context"
+ "testing"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/filter"
+ "github.com/rclone/rclone/fs/list"
+ "github.com/rclone/rclone/fstest"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// testListDirSorted is integration testing code in fs/list/list.go
+// which can't be tested there due to import loops.
+func testListDirSorted(t *testing.T, listFn func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error)) {
+ r := fstest.NewRun(t)
+
+ ctx := context.Background()
+ fi := filter.GetConfig(ctx)
+ fi.Opt.MaxSize = 10
+ defer func() {
+ fi.Opt.MaxSize = -1
+ }()
+
+ files := []fstest.Item{
+ r.WriteObject(context.Background(), "a.txt", "hello world", t1),
+ r.WriteObject(context.Background(), "zend.txt", "hello", t1),
+ r.WriteObject(context.Background(), "sub dir/hello world", "hello world", t1),
+ r.WriteObject(context.Background(), "sub dir/hello world2", "hello world", t1),
+ r.WriteObject(context.Background(), "sub dir/ignore dir/.ignore", "-", t1),
+ r.WriteObject(context.Background(), "sub dir/ignore dir/should be ignored", "to ignore", t1),
+ r.WriteObject(context.Background(), "sub dir/sub sub dir/hello world3", "hello world", t1),
+ }
+ r.CheckRemoteItems(t, files...)
+ var items fs.DirEntries
+ var err error
+
+ // Turn the DirEntry into a name, ending with a / if it is a
+ // dir
+ str := func(i int) string {
+ item := items[i]
+ name := item.Remote()
+ switch item.(type) {
+ case fs.Object:
+ case fs.Directory:
+ name += "/"
+ default:
+ t.Fatalf("Unknown type %+v", item)
+ }
+ return name
+ }
+
+ items, err = listFn(context.Background(), r.Fremote, true, "")
+ require.NoError(t, err)
+ require.Len(t, items, 3)
+ assert.Equal(t, "a.txt", str(0))
+ assert.Equal(t, "sub dir/", str(1))
+ assert.Equal(t, "zend.txt", str(2))
+
+ items, err = listFn(context.Background(), r.Fremote, false, "")
+ require.NoError(t, err)
+ require.Len(t, items, 2)
+ assert.Equal(t, "sub dir/", str(0))
+ assert.Equal(t, "zend.txt", str(1))
+
+ items, err = listFn(context.Background(), r.Fremote, true, "sub dir")
+ require.NoError(t, err)
+ require.Len(t, items, 4)
+ assert.Equal(t, "sub dir/hello world", str(0))
+ assert.Equal(t, "sub dir/hello world2", str(1))
+ assert.Equal(t, "sub dir/ignore dir/", str(2))
+ assert.Equal(t, "sub dir/sub sub dir/", str(3))
+
+ items, err = listFn(context.Background(), r.Fremote, false, "sub dir")
+ require.NoError(t, err)
+ require.Len(t, items, 2)
+ assert.Equal(t, "sub dir/ignore dir/", str(0))
+ assert.Equal(t, "sub dir/sub sub dir/", str(1))
+
+ // testing ignore file
+ fi.Opt.ExcludeFile = []string{".ignore"}
+
+ items, err = listFn(context.Background(), r.Fremote, false, "sub dir")
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+ assert.Equal(t, "sub dir/sub sub dir/", str(0))
+
+ items, err = listFn(context.Background(), r.Fremote, false, "sub dir/ignore dir")
+ require.NoError(t, err)
+ require.Len(t, items, 0)
+
+ items, err = listFn(context.Background(), r.Fremote, true, "sub dir/ignore dir")
+ require.NoError(t, err)
+ require.Len(t, items, 2)
+ assert.Equal(t, "sub dir/ignore dir/.ignore", str(0))
+ assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1))
+
+ fi.Opt.ExcludeFile = nil
+ items, err = listFn(context.Background(), r.Fremote, false, "sub dir/ignore dir")
+ require.NoError(t, err)
+ require.Len(t, items, 2)
+ assert.Equal(t, "sub dir/ignore dir/.ignore", str(0))
+ assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1))
+}
+
+// TestListDirSorted is integration testing code in fs/list/list.go
+// which can't be tested there due to import loops.
+func TestListDirSorted(t *testing.T) {
+ testListDirSorted(t, list.DirSorted)
+}
+
+// TestListDirSortedFn is integration testing code in fs/list/list.go
+// which can't be tested there due to import loops.
+func TestListDirSortedFn(t *testing.T) {
+ listFn := func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
+ callback := func(newEntries fs.DirEntries) error {
+ entries = append(entries, newEntries...)
+ return nil
+ }
+ err = list.DirSortedFn(ctx, f, includeAll, dir, callback, nil)
+ return entries, err
+ }
+ testListDirSorted(t, listFn)
+}
diff --git a/fs/operations/logger.go b/fs/operations/logger.go
new file mode 100644
index 0000000..0d48220
--- /dev/null
+++ b/fs/operations/logger.go
@@ -0,0 +1,384 @@
+package operations
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ mutex "sync"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/spf13/pflag"
+)
+
+// Sigil represents the rune (-+=*!?) used by Logger to categorize files by their match/differ/missing status.
+type Sigil rune
+
+// String converts sigil to more human-readable string
+func (sigil Sigil) String() string {
+ switch sigil {
+ case '-':
+ return "MissingOnSrc"
+ case '+':
+ return "MissingOnDst"
+ case '=':
+ return "Match"
+ case '*':
+ return "Differ"
+ case '!':
+ return "Error"
+ // case '.':
+ // return "Completed"
+ case '?':
+ return "Other"
+ }
+ return "unknown"
+}
+
+// Writer directs traffic from sigil -> LoggerOpt.Writer
+func (sigil Sigil) Writer(opt LoggerOpt) io.Writer {
+ switch sigil {
+ case '-':
+ return opt.MissingOnSrc
+ case '+':
+ return opt.MissingOnDst
+ case '=':
+ return opt.Match
+ case '*':
+ return opt.Differ
+ case '!':
+ return opt.Error
+ }
+ return nil
+}
+
+// Sigil constants
+const (
+ MissingOnSrc Sigil = '-'
+ MissingOnDst Sigil = '+'
+ Match Sigil = '='
+ Differ Sigil = '*'
+ TransferError Sigil = '!'
+ Other Sigil = '?' // reserved but not currently used
+)
+
+// LoggerFn uses fs.DirEntry instead of fs.Object so it can include Dirs
+// For LoggerFn example, see bisync.WriteResults() or sync.SyncLoggerFn()
+// Usage example: s.logger(ctx, operations.Differ, src, dst, nil)
+type LoggerFn func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error)
+type loggerContextKey struct{}
+type loggerOptContextKey struct{}
+
+var loggerKey = loggerContextKey{}
+var loggerOptKey = loggerOptContextKey{}
+
+// LoggerOpt contains options for the Sync Logger functions
+// TODO: refactor Check in here too?
+type LoggerOpt struct {
+ // Fdst, Fsrc fs.Fs // fses to check
+ // Check checkFn // function to use for checking
+ // OneWay bool // one way only?
+ LoggerFn LoggerFn // function to use for logging
+ Combined io.Writer // a file with file names with leading sigils
+ MissingOnSrc io.Writer // files only in the destination
+ MissingOnDst io.Writer // files only in the source
+ Match io.Writer // matching files
+ Differ io.Writer // differing files
+ Error io.Writer // files with errors of some kind
+ DestAfter io.Writer // files that exist on the destination post-sync
+ JSON *bytes.Buffer // used by bisync to read/write struct as JSON
+ DeleteModeOff bool //affects whether Logger expects MissingOnSrc to be deleted
+
+ // lsf options for destAfter
+ ListFormat ListFormat
+ JSONOpt ListJSONOpt
+ LJ *listJSON
+ Format string
+ TimeFormat string
+ Separator string
+ DirSlash bool
+ // Recurse bool
+ HashType hash.Type
+ FilesOnly bool
+ DirsOnly bool
+ Csv bool
+ Absolute bool
+}
+
+// NewDefaultLoggerFn creates a logger function that writes the sigil and path to configured files that match the sigil
+func NewDefaultLoggerFn(opt *LoggerOpt) LoggerFn {
+ var lock mutex.Mutex
+
+ return func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {
+ lock.Lock()
+ defer lock.Unlock()
+
+ if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil {
+ opt.PrintDestAfter(ctx, sigil, src, dst, err)
+ return
+ }
+
+ _, srcOk := src.(fs.Object)
+ _, dstOk := dst.(fs.Object)
+ var filename string
+ if !srcOk && !dstOk {
+ return
+ } else if srcOk && !dstOk {
+ filename = src.String()
+ } else {
+ filename = dst.String()
+ }
+
+ if sigil.Writer(*opt) != nil {
+ SyncFprintf(sigil.Writer(*opt), "%s\n", filename)
+ }
+ if opt.Combined != nil {
+ SyncFprintf(opt.Combined, "%c %s\n", sigil, filename)
+ fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename)
+ }
+ if opt.DestAfter != nil {
+ opt.PrintDestAfter(ctx, sigil, src, dst, err)
+ }
+ }
+}
+
+// WithLogger stores logger in ctx and returns a copy of ctx in which loggerKey = logger
+func WithLogger(ctx context.Context, logger LoggerFn) context.Context {
+ return context.WithValue(ctx, loggerKey, logger)
+}
+
+// WithLoggerOpt stores loggerOpt in ctx and returns a copy of ctx in which loggerOptKey = loggerOpt
+func WithLoggerOpt(ctx context.Context, loggerOpt LoggerOpt) context.Context {
+ return context.WithValue(ctx, loggerOptKey, loggerOpt)
+}
+
+// GetLogger attempts to retrieve LoggerFn from context, returns it if found, otherwise returns no-op function
+func GetLogger(ctx context.Context) (LoggerFn, bool) {
+ logger, ok := ctx.Value(loggerKey).(LoggerFn)
+ if !ok {
+ logger = func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {}
+ }
+ return logger, ok
+}
+
+// GetLoggerOpt attempts to retrieve LoggerOpt from context, returns it if found, otherwise returns NewLoggerOpt()
+func GetLoggerOpt(ctx context.Context) LoggerOpt {
+ loggerOpt, ok := ctx.Value(loggerOptKey).(LoggerOpt)
+ if ok {
+ return loggerOpt
+ }
+ return NewLoggerOpt()
+}
+
+// WithSyncLogger starts a new logger with the options passed in and saves it to ctx for retrieval later
+func WithSyncLogger(ctx context.Context, opt LoggerOpt) context.Context {
+ ctx = WithLoggerOpt(ctx, opt)
+ return WithLogger(ctx, func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {
+ if opt.LoggerFn != nil {
+ opt.LoggerFn(ctx, sigil, src, dst, err)
+ } else {
+ SyncFprintf(opt.Combined, "%c %s\n", sigil, dst.Remote())
+ }
+ })
+}
+
+// NewLoggerOpt returns a new LoggerOpt struct with defaults
+func NewLoggerOpt() LoggerOpt {
+ opt := LoggerOpt{
+ Combined: new(bytes.Buffer),
+ MissingOnSrc: new(bytes.Buffer),
+ MissingOnDst: new(bytes.Buffer),
+ Match: new(bytes.Buffer),
+ Differ: new(bytes.Buffer),
+ Error: new(bytes.Buffer),
+ DestAfter: new(bytes.Buffer),
+ JSON: new(bytes.Buffer),
+ }
+ return opt
+}
+
+// Winner predicts which side (src or dst) should end up winning out on the dst.
+type Winner struct {
+ Obj fs.DirEntry // the object that should exist on dst post-sync, if any
+ Side string // whether the winning object was from the src or dst
+ Err error // whether there's an error preventing us from predicting winner correctly (not whether there was a sync error more generally)
+}
+
+// WinningSide can be called in a LoggerFn to predict what the dest will look like post-sync
+//
+// This attempts to account for every case in which dst (intentionally) does not match src after a sync.
+//
+// Known issues / cases we can't confidently predict yet:
+//
+// --max-duration / CutoffModeHard
+// --compare-dest / --copy-dest (because equal() is called multiple times for the same file)
+// server-side moves of an entire dir at once (because we never get the individual file objects in the dir)
+// High-level retries, because there would be dupes (use --retries 1 to disable)
+// Possibly some error scenarios
+func WinningSide(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) Winner {
+ winner := Winner{nil, "none", nil}
+ opt := GetLoggerOpt(ctx)
+ ci := fs.GetConfig(ctx)
+
+ if err == fs.ErrorIsDir {
+ winner.Err = err
+ if sigil == MissingOnSrc {
+ if (opt.DeleteModeOff || ci.DryRun) && dst != nil {
+ winner.Obj = dst
+ winner.Side = "dst" // whatever's on dst will remain so after DryRun
+ return winner
+ }
+ return winner // none, because dst should just get deleted
+ }
+ if sigil == MissingOnDst && ci.DryRun {
+ return winner // none, because it does not currently exist on dst, and will still not exist after DryRun
+ } else if ci.DryRun && dst != nil {
+ winner.Obj = dst
+ winner.Side = "dst"
+ } else if src != nil {
+ winner.Obj = src
+ winner.Side = "src"
+ }
+ return winner
+ }
+
+ _, srcOk := src.(fs.Object)
+ _, dstOk := dst.(fs.Object)
+ if !srcOk && !dstOk {
+ return winner // none, because we don't have enough info to continue.
+ }
+
+ switch sigil {
+ case MissingOnSrc:
+ if opt.DeleteModeOff || ci.DryRun { // i.e. it's a copy, not sync (or it's a DryRun)
+ winner.Obj = dst
+ winner.Side = "dst" // whatever's on dst will remain so after DryRun
+ return winner
+ }
+ return winner // none, because dst should just get deleted
+ case Match, Differ, MissingOnDst:
+ if sigil == MissingOnDst && ci.DryRun {
+ return winner // none, because it does not currently exist on dst, and will still not exist after DryRun
+ }
+ winner.Obj = src
+ winner.Side = "src" // presume dst will end up matching src unless changed below
+ if sigil == Match && (ci.SizeOnly || ci.CheckSum || ci.IgnoreSize || ci.UpdateOlder || ci.NoUpdateModTime) {
+ winner.Obj = dst
+ winner.Side = "dst" // ignore any differences with src because of user flags
+ }
+ if ci.IgnoreTimes {
+ winner.Obj = src
+ winner.Side = "src" // copy src to dst unconditionally
+ }
+ if (sigil == Match || sigil == Differ) && (ci.IgnoreExisting || ci.Immutable) {
+ winner.Obj = dst
+ winner.Side = "dst" // dst should remain unchanged if it already exists (and we know it does because it's Match or Differ)
+ }
+ if ci.DryRun {
+ winner.Obj = dst
+ winner.Side = "dst" // dst should remain unchanged after DryRun (note that we handled MissingOnDst earlier)
+ }
+ return winner
+ case TransferError:
+ winner.Obj = dst
+ winner.Side = "dst" // usually, dst should not change if there's an error
+ if dst == nil {
+ winner.Obj = src
+ winner.Side = "src" // but if for some reason we have a src and not a dst, go with it
+ }
+ if winner.Obj != nil {
+ if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, errors.New("max transfer duration reached as set by --max-duration")) {
+ winner.Err = err // we can't confidently predict what survives if CutoffModeHard
+ }
+ return winner // we know at least one of the objects
+ }
+ }
+ // should only make it this far if it's TransferError and both src and dst are nil
+ winner.Side = "none"
+ winner.Err = fmt.Errorf("unknown case -- can't determine winner. %v", err)
+ fs.Debugf(winner.Obj, "%v", winner.Err)
+ return winner
+}
+
+// SetListFormat sets opt.ListFormat for destAfter
+// TODO: possibly refactor duplicate code from cmd/lsf, where this is mostly copied from
+func (opt *LoggerOpt) SetListFormat(ctx context.Context, cmdFlags *pflag.FlagSet) {
+ // Work out if the separatorFlag was supplied or not
+ separatorFlag := cmdFlags.Lookup("separator")
+ separatorFlagSupplied := separatorFlag != nil && separatorFlag.Changed
+ // Default the separator to , if using CSV
+ if opt.Csv && !separatorFlagSupplied {
+ opt.Separator = ","
+ }
+
+ var list ListFormat
+ list.SetSeparator(opt.Separator)
+ list.SetCSV(opt.Csv)
+ list.SetDirSlash(opt.DirSlash)
+ list.SetAbsolute(opt.Absolute)
+ var JSONOpt = ListJSONOpt{
+ NoModTime: true,
+ NoMimeType: true,
+ DirsOnly: opt.DirsOnly,
+ FilesOnly: opt.FilesOnly,
+ // Recurse: opt.Recurse,
+ }
+
+ for _, char := range opt.Format {
+ switch char {
+ case 'p':
+ list.AddPath()
+ case 't':
+ list.AddModTime(opt.TimeFormat)
+ JSONOpt.NoModTime = false
+ case 's':
+ list.AddSize()
+ case 'h':
+ list.AddHash(opt.HashType)
+ JSONOpt.ShowHash = true
+ JSONOpt.HashTypes = []string{opt.HashType.String()}
+ case 'i':
+ list.AddID()
+ case 'm':
+ list.AddMimeType()
+ JSONOpt.NoMimeType = false
+ case 'e':
+ list.AddEncrypted()
+ JSONOpt.ShowEncrypted = true
+ case 'o':
+ list.AddOrigID()
+ JSONOpt.ShowOrigIDs = true
+ case 'T':
+ list.AddTier()
+ case 'M':
+ list.AddMetadata()
+ JSONOpt.Metadata = true
+ default:
+ fs.Errorf(nil, "unknown format character %q", char)
+ }
+ }
+ opt.ListFormat = list
+ opt.JSONOpt = JSONOpt
+}
+
+// NewListJSON makes a new *listJSON for destAfter
+func (opt *LoggerOpt) NewListJSON(ctx context.Context, fdst fs.Fs, remote string) {
+ opt.LJ, _ = newListJSON(ctx, fdst, remote, &opt.JSONOpt)
+ //fs.Debugf(nil, "%v", opt.LJ)
+}
+
+// JSONEntry returns a *ListJSONItem for destAfter
+func (opt *LoggerOpt) JSONEntry(ctx context.Context, entry fs.DirEntry) (*ListJSONItem, error) {
+ return opt.LJ.entry(ctx, entry)
+}
+
+// PrintDestAfter writes a *ListJSONItem to opt.DestAfter
+func (opt *LoggerOpt) PrintDestAfter(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {
+ entry := WinningSide(ctx, sigil, src, dst, err)
+ if entry.Obj != nil {
+ JSONEntry, _ := opt.JSONEntry(ctx, entry.Obj)
+ _, _ = fmt.Fprintln(opt.DestAfter, opt.ListFormat.Format(JSONEntry))
+ }
+}
diff --git a/fs/operations/lsjson.go b/fs/operations/lsjson.go
new file mode 100644
index 0000000..8e714ef
--- /dev/null
+++ b/fs/operations/lsjson.go
@@ -0,0 +1,348 @@
+package operations
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "path"
+ "strings"
+ "time"
+
+ "github.com/rclone/rclone/backend/crypt"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/walk"
+)
+
+// ListJSONItem in the struct which gets marshalled for each line
+type ListJSONItem struct {
+ Path string
+ Name string
+ EncryptedPath string `json:",omitempty"`
+ Encrypted string `json:",omitempty"`
+ Size int64
+ MimeType string `json:",omitempty"`
+ ModTime Timestamp //`json:",omitempty"`
+ IsDir bool
+ Hashes map[string]string `json:",omitempty"`
+ ID string `json:",omitempty"`
+ OrigID string `json:",omitempty"`
+ Tier string `json:",omitempty"`
+ IsBucket bool `json:",omitempty"`
+ Metadata fs.Metadata `json:",omitempty"`
+}
+
+// Timestamp a time in the provided format
+type Timestamp struct {
+ When time.Time
+ Format string
+}
+
+// MarshalJSON turns a Timestamp into JSON
+func (t Timestamp) MarshalJSON() (out []byte, err error) {
+ if t.When.IsZero() {
+ return []byte(`""`), nil
+ }
+ return []byte(`"` + t.When.Format(t.Format) + `"`), nil
+}
+
+// Returns a time format for the given precision
+func formatForPrecision(precision time.Duration) string {
+ switch {
+ case precision <= time.Nanosecond:
+ return "2006-01-02T15:04:05.000000000Z07:00"
+ case precision <= 10*time.Nanosecond:
+ return "2006-01-02T15:04:05.00000000Z07:00"
+ case precision <= 100*time.Nanosecond:
+ return "2006-01-02T15:04:05.0000000Z07:00"
+ case precision <= time.Microsecond:
+ return "2006-01-02T15:04:05.000000Z07:00"
+ case precision <= 10*time.Microsecond:
+ return "2006-01-02T15:04:05.00000Z07:00"
+ case precision <= 100*time.Microsecond:
+ return "2006-01-02T15:04:05.0000Z07:00"
+ case precision <= time.Millisecond:
+ return "2006-01-02T15:04:05.000Z07:00"
+ case precision <= 10*time.Millisecond:
+ return "2006-01-02T15:04:05.00Z07:00"
+ case precision <= 100*time.Millisecond:
+ return "2006-01-02T15:04:05.0Z07:00"
+ }
+ return time.RFC3339
+}
+
+// ListJSONOpt describes the options for ListJSON
+type ListJSONOpt struct {
+ Recurse bool `json:"recurse"`
+ NoModTime bool `json:"noModTime"`
+ NoMimeType bool `json:"noMimeType"`
+ ShowEncrypted bool `json:"showEncrypted"`
+ ShowOrigIDs bool `json:"showOrigIDs"`
+ ShowHash bool `json:"showHash"`
+ DirsOnly bool `json:"dirsOnly"`
+ FilesOnly bool `json:"filesOnly"`
+ Metadata bool `json:"metadata"`
+ HashTypes []string `json:"hashTypes"` // hash types to show if ShowHash is set, e.g. "MD5", "SHA-1"
+}
+
+// state for ListJson
+type listJSON struct {
+ fsrc fs.Fs
+ remote string
+ format string
+ opt *ListJSONOpt
+ cipher *crypt.Cipher
+ hashTypes []hash.Type
+ dirs bool
+ files bool
+ canGetTier bool
+ isBucket bool
+ showHash bool
+}
+
+func newListJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt) (*listJSON, error) {
+ lj := &listJSON{
+ fsrc: fsrc,
+ remote: remote,
+ opt: opt,
+ dirs: true,
+ files: true,
+ }
+ // Dirs Files
+ // !FilesOnly,!DirsOnly true true
+ // !FilesOnly,DirsOnly true false
+ // FilesOnly,!DirsOnly false true
+ // FilesOnly,DirsOnly true true
+ if !opt.FilesOnly && opt.DirsOnly {
+ lj.files = false
+ } else if opt.FilesOnly && !opt.DirsOnly {
+ lj.dirs = false
+ }
+ if opt.ShowEncrypted {
+ fsInfo, _, _, config, err := fs.ConfigFs(fs.ConfigStringFull(fsrc))
+ if err != nil {
+ return nil, fmt.Errorf("ListJSON failed to load config for crypt remote: %w", err)
+ }
+ if fsInfo.Name != "crypt" {
+ return nil, errors.New("the remote needs to be of type \"crypt\"")
+ }
+ lj.cipher, err = crypt.NewCipher(config)
+ if err != nil {
+ return nil, fmt.Errorf("ListJSON failed to make new crypt remote: %w", err)
+ }
+ }
+ features := fsrc.Features()
+ lj.canGetTier = features.GetTier
+ lj.format = formatForPrecision(fsrc.Precision())
+ lj.isBucket = features.BucketBased && remote == "" && fsrc.Root() == "" // if bucket-based remote listing the root mark directories as buckets
+ lj.showHash = opt.ShowHash
+ lj.hashTypes = fsrc.Hashes().Array()
+ if len(opt.HashTypes) != 0 {
+ lj.showHash = true
+ lj.hashTypes = []hash.Type{}
+ for _, hashType := range opt.HashTypes {
+ var ht hash.Type
+ err := ht.Set(hashType)
+ if err != nil {
+ return nil, err
+ }
+ lj.hashTypes = append(lj.hashTypes, ht)
+ }
+ }
+ return lj, nil
+}
+
+// Convert a single entry to JSON
+//
+// It may return nil if there is no entry to return
+func (lj *listJSON) entry(ctx context.Context, entry fs.DirEntry) (*ListJSONItem, error) {
+ switch entry.(type) {
+ case fs.Directory:
+ if lj.opt.FilesOnly {
+ return nil, nil
+ }
+ case fs.Object:
+ if lj.opt.DirsOnly {
+ return nil, nil
+ }
+ default:
+ fs.Errorf(nil, "Unknown type %T in listing", entry)
+ }
+
+ item := &ListJSONItem{
+ Path: entry.Remote(),
+ Name: path.Base(entry.Remote()),
+ Size: entry.Size(),
+ }
+ if entry.Remote() == "" {
+ item.Name = ""
+ }
+ if !lj.opt.NoModTime {
+ item.ModTime = Timestamp{When: entry.ModTime(ctx), Format: lj.format}
+ }
+ if !lj.opt.NoMimeType {
+ item.MimeType = fs.MimeTypeDirEntry(ctx, entry)
+ }
+ if lj.cipher != nil {
+ switch entry.(type) {
+ case fs.Directory:
+ item.EncryptedPath = lj.cipher.EncryptDirName(entry.Remote())
+ case fs.Object:
+ item.EncryptedPath = lj.cipher.EncryptFileName(entry.Remote())
+ default:
+ fs.Errorf(nil, "Unknown type %T in listing", entry)
+ }
+ item.Encrypted = path.Base(item.EncryptedPath)
+ }
+ if lj.opt.Metadata {
+ metadata, err := fs.GetMetadata(ctx, entry)
+ if err != nil {
+ fs.Errorf(entry, "Failed to read metadata: %v", err)
+ } else if metadata != nil {
+ item.Metadata = metadata
+ }
+ }
+ if do, ok := entry.(fs.IDer); ok {
+ item.ID = do.ID()
+ }
+ if o, ok := entry.(fs.Object); lj.opt.ShowOrigIDs && ok {
+ if do, ok := fs.UnWrapObject(o).(fs.IDer); ok {
+ item.OrigID = do.ID()
+ }
+ }
+ switch x := entry.(type) {
+ case fs.Directory:
+ item.IsDir = true
+ item.IsBucket = lj.isBucket
+ case fs.Object:
+ item.IsDir = false
+ if lj.showHash {
+ item.Hashes = make(map[string]string)
+ for _, hashType := range lj.hashTypes {
+ hash, err := x.Hash(ctx, hashType)
+ if err != nil {
+ fs.Errorf(x, "Failed to read hash: %v", err)
+ } else if hash != "" {
+ item.Hashes[hashType.String()] = hash
+ }
+ }
+ }
+ if lj.canGetTier {
+ if do, ok := x.(fs.GetTierer); ok {
+ item.Tier = do.GetTier()
+ }
+ }
+ default:
+ fs.Errorf(nil, "Unknown type %T in listing in ListJSON", entry)
+ }
+ return item, nil
+}
+
+// ListJSON lists fsrc using the options in opt calling callback for each item
+func ListJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt, callback func(*ListJSONItem) error) error {
+ lj, err := newListJSON(ctx, fsrc, remote, opt)
+ if err != nil {
+ return err
+ }
+ err = walk.ListR(ctx, fsrc, remote, false, ConfigMaxDepth(ctx, lj.opt.Recurse), walk.ListAll, func(entries fs.DirEntries) (err error) {
+ for _, entry := range entries {
+ item, err := lj.entry(ctx, entry)
+ if err != nil {
+ return fmt.Errorf("creating entry failed in ListJSON: %w", err)
+ }
+ if item != nil {
+ err = callback(item)
+ if err != nil {
+ return fmt.Errorf("callback failed in ListJSON: %w", err)
+ }
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return fmt.Errorf("error in ListJSON: %w", err)
+ }
+ return nil
+}
+
+// StatJSON returns a single JSON stat entry for the fsrc, remote path
+//
+// The item returned may be nil if it is not found or excluded with DirsOnly/FilesOnly
+func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt) (item *ListJSONItem, err error) {
+ // FIXME this could me more efficient we had a new primitive
+ // NewDirEntry() which returned an Object or a Directory
+ lj, err := newListJSON(ctx, fsrc, remote, opt)
+ if err != nil {
+ return nil, err
+ }
+
+ // Root is always a directory. When we have a NewDirEntry
+ // primitive we need to call it, but for now this will do.
+ if remote == "" {
+ if !lj.dirs {
+ return nil, nil
+ }
+ // Check the root directory exists
+ entries, err := fsrc.List(ctx, "")
+ accounting.Stats(ctx).Listed(int64(len(entries)))
+ if err != nil {
+ return nil, err
+ }
+ return lj.entry(ctx, fs.NewDir("", time.Now()))
+ }
+
+ // Could be a file or a directory here
+ if lj.files && !strings.HasSuffix(remote, "/") {
+ // NewObject can return the sentinel errors ErrorObjectNotFound or ErrorIsDir
+ // ErrorObjectNotFound can mean the source is a directory or not found
+ obj, err := fsrc.NewObject(ctx, remote)
+ if err == fs.ErrorObjectNotFound {
+ if !lj.dirs {
+ return nil, nil
+ }
+ } else if err == fs.ErrorIsDir {
+ if !lj.dirs {
+ return nil, nil
+ }
+ // This could return a made up ListJSONItem here
+ // but that wouldn't have the IDs etc in
+ } else if err != nil {
+ if !lj.dirs {
+ return nil, err
+ }
+ } else {
+ return lj.entry(ctx, obj)
+ }
+ }
+ // Must be a directory here
+ //
+ // Remove trailing / as rclone listings won't have them
+ remote = strings.TrimRight(remote, "/")
+ parent := path.Dir(remote)
+ if parent == "." || parent == "/" {
+ parent = ""
+ }
+ entries, err := fsrc.List(ctx, parent)
+ accounting.Stats(ctx).Listed(int64(len(entries)))
+ if err == fs.ErrorDirNotFound {
+ return nil, nil
+ } else if err != nil {
+ return nil, err
+ }
+ equal := func(a, b string) bool { return a == b }
+ if fsrc.Features().CaseInsensitive {
+ equal = strings.EqualFold
+ }
+ var foundEntry fs.DirEntry
+ for _, entry := range entries {
+ if equal(entry.Remote(), remote) {
+ foundEntry = entry
+ break
+ }
+ }
+ if foundEntry == nil {
+ return nil, nil
+ }
+ return lj.entry(ctx, foundEntry)
+}
diff --git a/fs/operations/lsjson_test.go b/fs/operations/lsjson_test.go
new file mode 100644
index 0000000..75d4728
--- /dev/null
+++ b/fs/operations/lsjson_test.go
@@ -0,0 +1,406 @@
+package operations_test
+
+import (
+ "context"
+ "sort"
+ "testing"
+ "time"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/fstest"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// Compare a and b in a file system independent way
+func compareListJSONItem(t *testing.T, a, b *operations.ListJSONItem, precision time.Duration) {
+ assert.Equal(t, a.Path, b.Path, "Path")
+ assert.Equal(t, a.Name, b.Name, "Name")
+ // assert.Equal(t, a.EncryptedPath, b.EncryptedPath, "EncryptedPath")
+ // assert.Equal(t, a.Encrypted, b.Encrypted, "Encrypted")
+ if !a.IsDir {
+ assert.Equal(t, a.Size, b.Size, "Size")
+ }
+ // assert.Equal(t, a.MimeType, a.Mib.MimeType, "MimeType")
+ if !a.IsDir {
+ fstest.AssertTimeEqualWithPrecision(t, "ListJSON", a.ModTime.When, b.ModTime.When, precision)
+ }
+ assert.Equal(t, a.IsDir, b.IsDir, "IsDir")
+ // assert.Equal(t, a.Hashes, a.b.Hashes, "Hashes")
+ // assert.Equal(t, a.ID, b.ID, "ID")
+ // assert.Equal(t, a.OrigID, a.b.OrigID, "OrigID")
+ // assert.Equal(t, a.Tier, b.Tier, "Tier")
+ // assert.Equal(t, a.IsBucket, a.Isb.IsBucket, "IsBucket")
+}
+
+func TestListJSON(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ file1 := r.WriteBoth(ctx, "file1", "file1", t1)
+ file2 := r.WriteBoth(ctx, "sub/file2", "sub/file2", t2)
+
+ r.CheckRemoteItems(t, file1, file2)
+ precision := fs.GetModifyWindow(ctx, r.Fremote)
+
+ for _, test := range []struct {
+ name string
+ remote string
+ opt operations.ListJSONOpt
+ want []*operations.ListJSONItem
+ }{
+ {
+ name: "Default",
+ opt: operations.ListJSONOpt{},
+ want: []*operations.ListJSONItem{{
+ Path: "file1",
+ Name: "file1",
+ Size: 5,
+ ModTime: operations.Timestamp{When: t1},
+ IsDir: false,
+ }, {
+ Path: "sub",
+ Name: "sub",
+ IsDir: true,
+ }},
+ }, {
+ name: "FilesOnly",
+ opt: operations.ListJSONOpt{
+ FilesOnly: true,
+ },
+ want: []*operations.ListJSONItem{{
+ Path: "file1",
+ Name: "file1",
+ Size: 5,
+ ModTime: operations.Timestamp{When: t1},
+ IsDir: false,
+ }},
+ }, {
+ name: "DirsOnly",
+ opt: operations.ListJSONOpt{
+ DirsOnly: true,
+ },
+ want: []*operations.ListJSONItem{{
+ Path: "sub",
+ Name: "sub",
+ IsDir: true,
+ }},
+ }, {
+ name: "Recurse",
+ opt: operations.ListJSONOpt{
+ Recurse: true,
+ },
+ want: []*operations.ListJSONItem{{
+ Path: "file1",
+ Name: "file1",
+ Size: 5,
+ ModTime: operations.Timestamp{When: t1},
+ IsDir: false,
+ }, {
+ Path: "sub",
+ Name: "sub",
+ IsDir: true,
+ }, {
+ Path: "sub/file2",
+ Name: "file2",
+ Size: 9,
+ ModTime: operations.Timestamp{When: t2},
+ IsDir: false,
+ }},
+ }, {
+ name: "SubDir",
+ remote: "sub",
+ opt: operations.ListJSONOpt{},
+ want: []*operations.ListJSONItem{{
+ Path: "sub/file2",
+ Name: "file2",
+ Size: 9,
+ ModTime: operations.Timestamp{When: t2},
+ IsDir: false,
+ }},
+ }, {
+ name: "NoModTime",
+ opt: operations.ListJSONOpt{
+ FilesOnly: true,
+ NoModTime: true,
+ },
+ want: []*operations.ListJSONItem{{
+ Path: "file1",
+ Name: "file1",
+ Size: 5,
+ ModTime: operations.Timestamp{When: time.Time{}},
+ IsDir: false,
+ }},
+ }, {
+ name: "NoMimeType",
+ opt: operations.ListJSONOpt{
+ FilesOnly: true,
+ NoMimeType: true,
+ },
+ want: []*operations.ListJSONItem{{
+ Path: "file1",
+ Name: "file1",
+ Size: 5,
+ ModTime: operations.Timestamp{When: t1},
+ IsDir: false,
+ }},
+ }, {
+ name: "ShowHash",
+ opt: operations.ListJSONOpt{
+ FilesOnly: true,
+ ShowHash: true,
+ },
+ want: []*operations.ListJSONItem{{
+ Path: "file1",
+ Name: "file1",
+ Size: 5,
+ ModTime: operations.Timestamp{When: t1},
+ IsDir: false,
+ }},
+ }, {
+ name: "HashTypes",
+ opt: operations.ListJSONOpt{
+ FilesOnly: true,
+ ShowHash: true,
+ HashTypes: []string{"MD5"},
+ },
+ want: []*operations.ListJSONItem{{
+ Path: "file1",
+ Name: "file1",
+ Size: 5,
+ ModTime: operations.Timestamp{When: t1},
+ IsDir: false,
+ }},
+ }, {
+ name: "Metadata",
+ opt: operations.ListJSONOpt{
+ FilesOnly: false,
+ Metadata: true,
+ },
+ want: []*operations.ListJSONItem{{
+ Path: "file1",
+ Name: "file1",
+ Size: 5,
+ ModTime: operations.Timestamp{When: t1},
+ IsDir: false,
+ }, {
+ Path: "sub",
+ Name: "sub",
+ IsDir: true,
+ }},
+ },
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ var got []*operations.ListJSONItem
+ require.NoError(t, operations.ListJSON(ctx, r.Fremote, test.remote, &test.opt, func(item *operations.ListJSONItem) error {
+ got = append(got, item)
+ return nil
+ }))
+ sort.Slice(got, func(i, j int) bool {
+ return got[i].Path < got[j].Path
+ })
+ require.Equal(t, len(test.want), len(got), "Wrong number of results")
+ for i := range test.want {
+ compareListJSONItem(t, test.want[i], got[i], precision)
+ if test.opt.NoMimeType {
+ assert.Equal(t, "", got[i].MimeType)
+ } else {
+ assert.NotEqual(t, "", got[i].MimeType)
+ }
+ if test.opt.Metadata {
+ features := r.Fremote.Features()
+ if features.ReadMetadata && !got[i].IsDir {
+ assert.Greater(t, len(got[i].Metadata), 0, "Expecting metadata for file")
+ }
+ if features.ReadDirMetadata && got[i].IsDir {
+ assert.Greater(t, len(got[i].Metadata), 0, "Expecting metadata for dir")
+ }
+ }
+ if test.opt.ShowHash {
+ hashes := got[i].Hashes
+ assert.NotNil(t, hashes)
+ if len(test.opt.HashTypes) > 0 && len(hashes) > 0 {
+ assert.Equal(t, 1, len(hashes))
+ }
+ if hashes["crc32"] != "" {
+ assert.Equal(t, "9ee760e5", hashes["crc32"])
+ }
+ if hashes["dropbox"] != "" {
+ assert.Equal(t, "f4d62afeaee6f35d3efdd8c66623360395165473bcc958f835343eb3f542f983", hashes["dropbox"])
+ }
+ if hashes["mailru"] != "" {
+ assert.Equal(t, "66696c6531000000000000000000000000000000", hashes["mailru"])
+ }
+ if hashes["md5"] != "" {
+ assert.Equal(t, "826e8142e6baabe8af779f5f490cf5f5", hashes["md5"])
+ }
+ if hashes["quickxor"] != "" {
+ assert.Equal(t, "6648031bca100300000000000500000000000000", hashes["quickxor"])
+ }
+ if hashes["sha1"] != "" {
+ assert.Equal(t, "60b27f004e454aca81b0480209cce5081ec52390", hashes["sha1"])
+ }
+ if hashes["sha256"] != "" {
+ assert.Equal(t, "c147efcfc2d7ea666a9e4f5187b115c90903f0fc896a56df9a6ef5d8f3fc9f31", hashes["sha256"])
+ }
+ if hashes["whirlpool"] != "" {
+ assert.Equal(t, "02fa11755b6470bfc5aab6d94cde5cf2939474fb5b0ebbf8ddf3d32bf06aa438eb92eac097047c02017dc1c317ee83fa8a2717ca4d544b4ee75b3231d1c466b0", hashes["whirlpool"])
+ }
+ } else {
+ assert.Nil(t, got[i].Hashes)
+ }
+ }
+ })
+ }
+}
+
+func TestStatJSON(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ file1 := r.WriteBoth(ctx, "file1", "file1", t1)
+ file2 := r.WriteBoth(ctx, "sub/file2", "sub/file2", t2)
+
+ r.CheckRemoteItems(t, file1, file2)
+ precision := fs.GetModifyWindow(ctx, r.Fremote)
+
+ for _, test := range []struct {
+ name string
+ remote string
+ opt operations.ListJSONOpt
+ want *operations.ListJSONItem
+ }{
+ {
+ name: "Root",
+ remote: "",
+ opt: operations.ListJSONOpt{},
+ want: &operations.ListJSONItem{
+ Path: "",
+ Name: "",
+ IsDir: true,
+ },
+ }, {
+ name: "RootFilesOnly",
+ remote: "",
+ opt: operations.ListJSONOpt{
+ FilesOnly: true,
+ },
+ want: nil,
+ }, {
+ name: "RootDirsOnly",
+ remote: "",
+ opt: operations.ListJSONOpt{
+ DirsOnly: true,
+ },
+ want: &operations.ListJSONItem{
+ Path: "",
+ Name: "",
+ IsDir: true,
+ },
+ }, {
+ name: "Dir",
+ remote: "sub",
+ opt: operations.ListJSONOpt{},
+ want: &operations.ListJSONItem{
+ Path: "sub",
+ Name: "sub",
+ IsDir: true,
+ },
+ }, {
+ name: "DirWithTrailingSlash",
+ remote: "sub/",
+ opt: operations.ListJSONOpt{},
+ want: &operations.ListJSONItem{
+ Path: "sub",
+ Name: "sub",
+ IsDir: true,
+ },
+ }, {
+ name: "File",
+ remote: "file1",
+ opt: operations.ListJSONOpt{},
+ want: &operations.ListJSONItem{
+ Path: "file1",
+ Name: "file1",
+ Size: 5,
+ ModTime: operations.Timestamp{When: t1},
+ IsDir: false,
+ },
+ }, {
+ name: "NotFound",
+ remote: "notfound",
+ opt: operations.ListJSONOpt{},
+ want: nil,
+ }, {
+ name: "DirFilesOnly",
+ remote: "sub",
+ opt: operations.ListJSONOpt{
+ FilesOnly: true,
+ },
+ want: nil,
+ }, {
+ name: "FileFilesOnly",
+ remote: "file1",
+ opt: operations.ListJSONOpt{
+ FilesOnly: true,
+ },
+ want: &operations.ListJSONItem{
+ Path: "file1",
+ Name: "file1",
+ Size: 5,
+ ModTime: operations.Timestamp{When: t1},
+ IsDir: false,
+ },
+ }, {
+ name: "NotFoundFilesOnly",
+ remote: "notfound",
+ opt: operations.ListJSONOpt{
+ FilesOnly: true,
+ },
+ want: nil,
+ }, {
+ name: "DirDirsOnly",
+ remote: "sub",
+ opt: operations.ListJSONOpt{
+ DirsOnly: true,
+ },
+ want: &operations.ListJSONItem{
+ Path: "sub",
+ Name: "sub",
+ IsDir: true,
+ },
+ }, {
+ name: "FileDirsOnly",
+ remote: "file1",
+ opt: operations.ListJSONOpt{
+ DirsOnly: true,
+ },
+ want: nil,
+ }, {
+ name: "NotFoundDirsOnly",
+ remote: "notfound",
+ opt: operations.ListJSONOpt{
+ DirsOnly: true,
+ },
+ want: nil,
+ },
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ got, err := operations.StatJSON(ctx, r.Fremote, test.remote, &test.opt)
+ require.NoError(t, err)
+ if test.want == nil {
+ assert.Nil(t, got)
+ return
+ }
+ require.NotNil(t, got)
+ compareListJSONItem(t, test.want, got, precision)
+ })
+ }
+
+ t.Run("RootNotFound", func(t *testing.T) {
+ f, err := fs.NewFs(ctx, r.FremoteName+"/notfound")
+ require.NoError(t, err)
+ _, err = operations.StatJSON(ctx, f, "", &operations.ListJSONOpt{})
+ // This should return an error except for bucket based remotes
+ assert.True(t, err != nil || f.Features().BucketBased, "Need an error for non bucket based backends")
+ })
+}
diff --git a/fs/operations/multithread.go b/fs/operations/multithread.go
new file mode 100644
index 0000000..9abd5f7
--- /dev/null
+++ b/fs/operations/multithread.go
@@ -0,0 +1,380 @@
+package operations
+
+import (
+ "bufio"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "time"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/lib/atexit"
+ "github.com/rclone/rclone/lib/multipart"
+ "github.com/rclone/rclone/lib/pool"
+ "golang.org/x/sync/errgroup"
+)
+
+const (
+ multithreadChunkSize = 64 << 10
+)
+
+// Return a boolean as to whether we should use multi thread copy for
+// this transfer
+func doMultiThreadCopy(ctx context.Context, f fs.Fs, src fs.Object) bool {
+ ci := fs.GetConfig(ctx)
+
+ // Disable multi thread if...
+
+ // ...it isn't configured
+ if ci.MultiThreadStreams <= 1 {
+ return false
+ }
+ // ...if the source doesn't support it
+ if src.Fs().Features().NoMultiThreading {
+ return false
+ }
+ // ...size of object is less than cutoff
+ if src.Size() < int64(ci.MultiThreadCutoff) {
+ return false
+ }
+ // ...destination doesn't support it
+ dstFeatures := f.Features()
+ if dstFeatures.OpenChunkWriter == nil && dstFeatures.OpenWriterAt == nil {
+ return false
+ }
+ // ...if --multi-thread-streams not in use and source and
+ // destination are both local
+ if !ci.MultiThreadSet && dstFeatures.IsLocal && src.Fs().Features().IsLocal {
+ return false
+ }
+ return true
+}
+
+// state for a multi-thread copy
+type multiThreadCopyState struct {
+ ctx context.Context
+ partSize int64
+ size int64
+ src fs.Object
+ acc *accounting.Account
+ numChunks int
+ noBuffering bool // set to read the input without buffering
+}
+
+// Copy a single chunk into place
+func (mc *multiThreadCopyState) copyChunk(ctx context.Context, chunk int, writer fs.ChunkWriter) (err error) {
+ defer func() {
+ if err != nil {
+ fs.Debugf(mc.src, "multi-thread copy: chunk %d/%d failed: %v", chunk+1, mc.numChunks, err)
+ }
+ }()
+ start := int64(chunk) * mc.partSize
+ if start >= mc.size {
+ return nil
+ }
+ end := min(start+mc.partSize, mc.size)
+ size := end - start
+
+ // Reserve the memory first so we don't open the source and wait for memory buffers for ages
+ var rw *pool.RW
+ if !mc.noBuffering {
+ rw = multipart.NewRW().Reserve(size)
+ defer fs.CheckClose(rw, &err)
+ }
+
+ fs.Debugf(mc.src, "multi-thread copy: chunk %d/%d (%d-%d) size %v starting", chunk+1, mc.numChunks, start, end, fs.SizeSuffix(size))
+
+ rc, err := Open(ctx, mc.src, &fs.RangeOption{Start: start, End: end - 1})
+ if err != nil {
+ return fmt.Errorf("multi-thread copy: failed to open source: %w", err)
+ }
+ defer fs.CheckClose(rc, &err)
+
+ var rs io.ReadSeeker
+ if mc.noBuffering {
+ // Read directly if we are sure we aren't going to seek
+ // and account with accounting
+ rc.SetAccounting(mc.acc.AccountRead)
+ rs = rc
+ } else {
+ // Read the chunk into buffered reader
+ _, err = io.CopyN(rw, rc, size)
+ if err != nil {
+ return fmt.Errorf("multi-thread copy: failed to read chunk: %w", err)
+ }
+ // Account as we go
+ rw.SetAccounting(mc.acc.AccountRead)
+ rs = rw
+ }
+
+ // Write the chunk
+ bytesWritten, err := writer.WriteChunk(ctx, chunk, rs)
+ if err != nil {
+ return fmt.Errorf("multi-thread copy: failed to write chunk: %w", err)
+ }
+
+ fs.Debugf(mc.src, "multi-thread copy: chunk %d/%d (%d-%d) size %v finished", chunk+1, mc.numChunks, start, end, fs.SizeSuffix(bytesWritten))
+ return nil
+}
+
+// Given a file size and a chunkSize
+// it returns the number of chunks, so that chunkSize * numChunks >= size
+func calculateNumChunks(size int64, chunkSize int64) int {
+ numChunks := size / chunkSize
+ if size%chunkSize != 0 {
+ numChunks++
+ }
+ return int(numChunks)
+}
+
+// Copy src to (f, remote) using streams download threads. It tries to use the OpenChunkWriter feature
+// and if that's not available it creates an adapter using OpenWriterAt
+func multiThreadCopy(ctx context.Context, f fs.Fs, remote string, src fs.Object, concurrency int, tr *accounting.Transfer, options ...fs.OpenOption) (newDst fs.Object, err error) {
+ openChunkWriter := f.Features().OpenChunkWriter
+ ci := fs.GetConfig(ctx)
+ noBuffering := false
+ usingOpenWriterAt := false
+ if openChunkWriter == nil {
+ openWriterAt := f.Features().OpenWriterAt
+ if openWriterAt == nil {
+ return nil, errors.New("multi-thread copy: neither OpenChunkWriter nor OpenWriterAt supported")
+ }
+ openChunkWriter = openChunkWriterFromOpenWriterAt(openWriterAt, int64(ci.MultiThreadChunkSize), int64(ci.MultiThreadWriteBufferSize), f)
+ // If we are using OpenWriterAt we don't seek the chunks so don't need to buffer
+ fs.Debugf(src, "multi-thread copy: disabling buffering because destination uses OpenWriterAt")
+ noBuffering = true
+ usingOpenWriterAt = true
+ } else if src.Fs().Features().IsLocal {
+ // If the source fs is local we don't need to buffer
+ fs.Debugf(src, "multi-thread copy: disabling buffering because source is local disk")
+ noBuffering = true
+ } else if f.Features().ChunkWriterDoesntSeek {
+ // If the destination Fs promises not to seek its chunks
+ // (except for retries) then we don't need buffering.
+ fs.Debugf(src, "multi-thread copy: disabling buffering because destination has set ChunkWriterDoesntSeek")
+ noBuffering = true
+ }
+
+ if src.Size() < 0 {
+ return nil, fmt.Errorf("multi-thread copy: can't copy unknown sized file")
+ }
+ if src.Size() == 0 {
+ return nil, fmt.Errorf("multi-thread copy: can't copy zero sized file")
+ }
+
+ info, chunkWriter, err := openChunkWriter(ctx, remote, src, options...)
+ if err != nil {
+ return nil, fmt.Errorf("multi-thread copy: failed to open chunk writer: %w", err)
+ }
+
+ uploadCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ uploadedOK := false
+ defer atexit.OnError(&err, func() {
+ cancel()
+ if info.LeavePartsOnError || uploadedOK {
+ return
+ }
+ fs.Debugf(src, "multi-thread copy: cancelling transfer on exit")
+ abortErr := chunkWriter.Abort(ctx)
+ if abortErr != nil {
+ fs.Debugf(src, "multi-thread copy: abort failed: %v", abortErr)
+ }
+ })()
+
+ if info.ChunkSize > src.Size() {
+ fs.Debugf(src, "multi-thread copy: chunk size %v was bigger than source file size %v", fs.SizeSuffix(info.ChunkSize), fs.SizeSuffix(src.Size()))
+ info.ChunkSize = src.Size()
+ }
+
+ // Use the backend concurrency if it is higher than --multi-thread-streams or if --multi-thread-streams wasn't set explicitly
+ if !ci.MultiThreadSet || info.Concurrency > concurrency {
+ fs.Debugf(src, "multi-thread copy: using backend concurrency of %d instead of --multi-thread-streams %d", info.Concurrency, concurrency)
+ concurrency = info.Concurrency
+ }
+
+ numChunks := calculateNumChunks(src.Size(), info.ChunkSize)
+ if concurrency > numChunks {
+ fs.Debugf(src, "multi-thread copy: number of streams %d was bigger than number of chunks %d", concurrency, numChunks)
+ concurrency = numChunks
+ }
+
+ if concurrency < 1 {
+ concurrency = 1
+ }
+
+ g, gCtx := errgroup.WithContext(uploadCtx)
+ g.SetLimit(concurrency)
+
+ mc := &multiThreadCopyState{
+ ctx: gCtx,
+ size: src.Size(),
+ src: src,
+ partSize: info.ChunkSize,
+ numChunks: numChunks,
+ noBuffering: noBuffering,
+ }
+
+ // Make accounting
+ mc.acc = tr.Account(gCtx, nil)
+
+ fs.Debugf(src, "Starting multi-thread copy with %d chunks of size %v with %v parallel streams", mc.numChunks, fs.SizeSuffix(mc.partSize), concurrency)
+ for chunk := range mc.numChunks {
+ // Fail fast, in case an errgroup managed function returns an error
+ if gCtx.Err() != nil {
+ break
+ }
+ chunk := chunk
+ g.Go(func() error {
+ return mc.copyChunk(gCtx, chunk, chunkWriter)
+ })
+ }
+
+ err = g.Wait()
+ if err != nil {
+ return nil, err
+ }
+ err = chunkWriter.Close(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("multi-thread copy: failed to close object after copy: %w", err)
+ }
+ uploadedOK = true // file is definitely uploaded OK so no need to abort
+
+ obj, err := f.NewObject(ctx, remote)
+ if err != nil {
+ return nil, fmt.Errorf("multi-thread copy: failed to find object after copy: %w", err)
+ }
+
+ // OpenWriterAt doesn't set metadata so we need to set it on completion
+ if usingOpenWriterAt {
+ setModTime := true
+ if ci.Metadata {
+ do, ok := obj.(fs.SetMetadataer)
+ if ok {
+ meta, err := fs.GetMetadataOptions(ctx, f, src, options)
+ if err != nil {
+ return nil, fmt.Errorf("multi-thread copy: failed to read metadata from source object: %w", err)
+ }
+ if _, foundMeta := meta["mtime"]; !foundMeta {
+ meta.Set("mtime", src.ModTime(ctx).Format(time.RFC3339Nano))
+ }
+ err = do.SetMetadata(ctx, meta)
+ if err != nil {
+ return nil, fmt.Errorf("multi-thread copy: failed to set metadata: %w", err)
+ }
+ setModTime = false
+ } else {
+ fs.Errorf(obj, "multi-thread copy: can't set metadata as SetMetadata isn't implemented in: %v", f)
+ }
+ }
+ if setModTime {
+ err = obj.SetModTime(ctx, src.ModTime(ctx))
+ switch err {
+ case nil, fs.ErrorCantSetModTime, fs.ErrorCantSetModTimeWithoutDelete:
+ default:
+ return nil, fmt.Errorf("multi-thread copy: failed to set modification time: %w", err)
+ }
+ }
+ }
+
+ fs.Debugf(src, "Finished multi-thread copy with %d parts of size %v", mc.numChunks, fs.SizeSuffix(mc.partSize))
+ return obj, nil
+}
+
+// writerAtChunkWriter converts a WriterAtCloser into a ChunkWriter
+type writerAtChunkWriter struct {
+ remote string
+ size int64
+ writerAt fs.WriterAtCloser
+ chunkSize int64
+ chunks int
+ writeBufferSize int64
+ f fs.Fs
+ closed bool
+}
+
+// WriteChunk writes chunkNumber from reader
+func (w *writerAtChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (int64, error) {
+ fs.Debugf(w.remote, "writing chunk %v", chunkNumber)
+
+ bytesToWrite := w.chunkSize
+ if chunkNumber == (w.chunks-1) && w.size%w.chunkSize != 0 {
+ bytesToWrite = w.size % w.chunkSize
+ }
+
+ var writer io.Writer = io.NewOffsetWriter(w.writerAt, int64(chunkNumber)*w.chunkSize)
+ if w.writeBufferSize > 0 {
+ writer = bufio.NewWriterSize(writer, int(w.writeBufferSize))
+ }
+ n, err := io.Copy(writer, reader)
+ if err != nil {
+ return -1, err
+ }
+ if n != bytesToWrite {
+ return -1, fmt.Errorf("expected to write %v bytes for chunk %v, but wrote %v bytes", bytesToWrite, chunkNumber, n)
+ }
+ // if we were buffering, flush to disk
+ switch w := writer.(type) {
+ case *bufio.Writer:
+ err = w.Flush()
+ if err != nil {
+ return -1, fmt.Errorf("multi-thread copy: flush failed: %w", err)
+ }
+ }
+ return n, nil
+}
+
+// Close the chunk writing
+func (w *writerAtChunkWriter) Close(ctx context.Context) error {
+ if w.closed {
+ return nil
+ }
+ w.closed = true
+ return w.writerAt.Close()
+}
+
+// Abort the chunk writing
+func (w *writerAtChunkWriter) Abort(ctx context.Context) error {
+ err := w.Close(ctx)
+ if err != nil {
+ fs.Errorf(w.remote, "multi-thread copy: failed to close file before aborting: %v", err)
+ }
+ obj, err := w.f.NewObject(ctx, w.remote)
+ if err != nil {
+ return fmt.Errorf("multi-thread copy: failed to find temp file when aborting chunk writer: %w", err)
+ }
+ return obj.Remove(ctx)
+}
+
+// openChunkWriterFromOpenWriterAt adapts an OpenWriterAtFn into an OpenChunkWriterFn using chunkSize and writeBufferSize
+func openChunkWriterFromOpenWriterAt(openWriterAt fs.OpenWriterAtFn, chunkSize int64, writeBufferSize int64, f fs.Fs) fs.OpenChunkWriterFn {
+ return func(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
+ ci := fs.GetConfig(ctx)
+
+ writerAt, err := openWriterAt(ctx, remote, src.Size())
+ if err != nil {
+ return info, nil, err
+ }
+
+ if writeBufferSize > 0 {
+ fs.Debugf(src.Remote(), "multi-thread copy: write buffer set to %v", writeBufferSize)
+ }
+
+ chunkWriter := &writerAtChunkWriter{
+ remote: remote,
+ size: src.Size(),
+ chunkSize: chunkSize,
+ chunks: calculateNumChunks(src.Size(), chunkSize),
+ writerAt: writerAt,
+ writeBufferSize: writeBufferSize,
+ f: f,
+ }
+ info = fs.ChunkWriterInfo{
+ ChunkSize: chunkSize,
+ Concurrency: ci.MultiThreadStreams,
+ }
+ return info, chunkWriter, nil
+ }
+}
diff --git a/fs/operations/multithread_test.go b/fs/operations/multithread_test.go
new file mode 100644
index 0000000..d3a07ae
--- /dev/null
+++ b/fs/operations/multithread_test.go
@@ -0,0 +1,333 @@
+package operations
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/object"
+ "github.com/rclone/rclone/fstest/mockfs"
+ "github.com/rclone/rclone/fstest/mockobject"
+ "github.com/rclone/rclone/lib/random"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fstest"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestDoMultiThreadCopy(t *testing.T) {
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
+ f, err := mockfs.NewFs(ctx, "potato", "", nil)
+ require.NoError(t, err)
+ src := mockobject.New("file.txt").WithContent([]byte(random.String(100)), mockobject.SeekModeNone)
+ srcFs, err := mockfs.NewFs(ctx, "sausage", "", nil)
+ require.NoError(t, err)
+ src.SetFs(srcFs)
+
+ oldStreams := ci.MultiThreadStreams
+ oldCutoff := ci.MultiThreadCutoff
+ oldIsSet := ci.MultiThreadSet
+ defer func() {
+ ci.MultiThreadStreams = oldStreams
+ ci.MultiThreadCutoff = oldCutoff
+ ci.MultiThreadSet = oldIsSet
+ }()
+
+ ci.MultiThreadStreams, ci.MultiThreadCutoff = 4, 50
+ ci.MultiThreadSet = false
+
+ nullWriterAt := func(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
+ panic("don't call me")
+ }
+ f.Features().OpenWriterAt = nullWriterAt
+
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
+
+ ci.MultiThreadStreams = 0
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
+ ci.MultiThreadStreams = 1
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
+ ci.MultiThreadStreams = 2
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
+
+ ci.MultiThreadCutoff = 200
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
+ ci.MultiThreadCutoff = 101
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
+ ci.MultiThreadCutoff = 100
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
+
+ f.Features().OpenWriterAt = nil
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
+ f.Features().OpenWriterAt = nullWriterAt
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
+
+ f.Features().IsLocal = true
+ srcFs.Features().IsLocal = true
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
+ ci.MultiThreadSet = true
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
+ ci.MultiThreadSet = false
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
+ srcFs.Features().IsLocal = false
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
+ srcFs.Features().IsLocal = true
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
+ f.Features().IsLocal = false
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
+ srcFs.Features().IsLocal = false
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
+
+ srcFs.Features().NoMultiThreading = true
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
+ srcFs.Features().NoMultiThreading = false
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
+}
+
+func TestMultithreadCalculateNumChunks(t *testing.T) {
+ for _, test := range []struct {
+ size int64
+ chunkSize int64
+ wantNumChunks int
+ }{
+ {size: 1, chunkSize: multithreadChunkSize, wantNumChunks: 1},
+ {size: 1 << 20, chunkSize: 1, wantNumChunks: 1 << 20},
+ {size: 1 << 20, chunkSize: 2, wantNumChunks: 1 << 19},
+ {size: (1 << 20) + 1, chunkSize: 2, wantNumChunks: (1 << 19) + 1},
+ {size: (1 << 20) - 1, chunkSize: 2, wantNumChunks: 1 << 19},
+ } {
+ t.Run(fmt.Sprintf("%+v", test), func(t *testing.T) {
+ mc := &multiThreadCopyState{}
+ mc.numChunks = calculateNumChunks(test.size, test.chunkSize)
+ assert.Equal(t, test.wantNumChunks, mc.numChunks)
+ })
+ }
+}
+
+// Skip if not multithread, returning the chunkSize otherwise
+func skipIfNotMultithread(ctx context.Context, t *testing.T, r *fstest.Run) int {
+ features := r.Fremote.Features()
+ if features.OpenChunkWriter == nil && features.OpenWriterAt == nil {
+ t.Skip("multithread writing not supported")
+ }
+
+ // Only support one hash for the local backend otherwise we end up spending a huge amount of CPU on hashing!
+ if r.Fremote.Features().IsLocal {
+ oldHashes := hash.SupportOnly([]hash.Type{r.Fremote.Hashes().GetOne()})
+ t.Cleanup(func() {
+ _ = hash.SupportOnly(oldHashes)
+ })
+ }
+
+ ci := fs.GetConfig(ctx)
+ chunkSize := int(ci.MultiThreadChunkSize)
+ if features.OpenChunkWriter != nil {
+ //OpenChunkWriter func(ctx context.Context, remote string, src ObjectInfo, options ...OpenOption) (info ChunkWriterInfo, writer ChunkWriter, err error)
+ const fileName = "chunksize-probe"
+ src := object.NewStaticObjectInfo(fileName, time.Now(), int64(100*fs.Mebi), true, nil, nil)
+ info, writer, err := features.OpenChunkWriter(ctx, fileName, src)
+ require.NoError(t, err)
+ chunkSize = int(info.ChunkSize)
+ err = writer.Abort(ctx)
+ require.NoError(t, err)
+ }
+ return chunkSize
+}
+
+func TestMultithreadCopy(t *testing.T) {
+ r := fstest.NewRun(t)
+ ctx := context.Background()
+ chunkSize := skipIfNotMultithread(ctx, t, r)
+ // Check every other transfer for metadata
+ checkMetadata := false
+ ctx, ci := fs.AddConfig(ctx)
+
+ for _, upload := range []bool{false, true} {
+ for _, test := range []struct {
+ size int
+ streams int
+ }{
+ {size: chunkSize*2 - 1, streams: 2},
+ {size: chunkSize * 2, streams: 2},
+ {size: chunkSize*2 + 1, streams: 2},
+ } {
+ checkMetadata = !checkMetadata
+ ci.Metadata = checkMetadata
+ fileName := fmt.Sprintf("test-multithread-copy-%v-%d-%d", upload, test.size, test.streams)
+ t.Run(fmt.Sprintf("upload=%v,size=%v,streams=%v", upload, test.size, test.streams), func(t *testing.T) {
+ if *fstest.SizeLimit > 0 && int64(test.size) > *fstest.SizeLimit {
+ t.Skipf("exceeded file size limit %d > %d", test.size, *fstest.SizeLimit)
+ }
+ var (
+ contents = random.String(test.size)
+ t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
+ file1 fstest.Item
+ src, dst fs.Object
+ err error
+ testMetadata = fs.Metadata{
+ // System metadata supported by all backends
+ "mtime": t1.Format(time.RFC3339Nano),
+ // User metadata
+ "potato": "jersey",
+ }
+ )
+
+ var fSrc, fDst fs.Fs
+ if upload {
+ file1 = r.WriteFile(fileName, contents, t1)
+ r.CheckRemoteItems(t)
+ r.CheckLocalItems(t, file1)
+ fDst, fSrc = r.Fremote, r.Flocal
+ } else {
+ file1 = r.WriteObject(ctx, fileName, contents, t1)
+ r.CheckRemoteItems(t, file1)
+ r.CheckLocalItems(t)
+ fDst, fSrc = r.Flocal, r.Fremote
+ }
+ src, err = fSrc.NewObject(ctx, fileName)
+ require.NoError(t, err)
+
+ do, canSetMetadata := src.(fs.SetMetadataer)
+ if checkMetadata && canSetMetadata {
+ // Set metadata on the source if required
+ err := do.SetMetadata(ctx, testMetadata)
+ if err == fs.ErrorNotImplemented {
+ canSetMetadata = false
+ } else {
+ require.NoError(t, err)
+ fstest.CheckEntryMetadata(ctx, t, r.Flocal, src, testMetadata)
+ }
+ }
+
+ accounting.GlobalStats().ResetCounters()
+ tr := accounting.GlobalStats().NewTransfer(src, nil)
+
+ defer func() {
+ tr.Done(ctx, err)
+ }()
+
+ dst, err = multiThreadCopy(ctx, fDst, fileName, src, test.streams, tr)
+ require.NoError(t, err)
+
+ assert.Equal(t, src.Size(), dst.Size())
+ assert.Equal(t, fileName, dst.Remote())
+ fstest.CheckListingWithPrecision(t, fSrc, []fstest.Item{file1}, nil, fs.GetModifyWindow(ctx, fDst, fSrc))
+ fstest.CheckListingWithPrecision(t, fDst, []fstest.Item{file1}, nil, fs.GetModifyWindow(ctx, fDst, fSrc))
+
+ if checkMetadata && canSetMetadata && fDst.Features().ReadMetadata {
+ fstest.CheckEntryMetadata(ctx, t, fDst, dst, testMetadata)
+ }
+
+ require.NoError(t, dst.Remove(ctx))
+ require.NoError(t, src.Remove(ctx))
+
+ })
+ }
+ }
+}
+
+type errorObject struct {
+ fs.Object
+ size int64
+ wg *sync.WaitGroup
+}
+
+// Open opens the file for read. Call Close() on the returned io.ReadCloser
+//
+// Remember this is called multiple times whenever the backend seeks (eg having read checksum)
+func (o errorObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+ fs.Debugf(nil, "Open with options = %v", options)
+ rc, err := o.Object.Open(ctx, options...)
+ if err != nil {
+ return nil, err
+ }
+ // Return an error reader for the second segment
+ for _, option := range options {
+ if ropt, ok := option.(*fs.RangeOption); ok {
+ end := ropt.End + 1
+ if end >= o.size {
+ // Give the other chunks a chance to start
+ time.Sleep(time.Second)
+ // Wait for chunks to upload first
+ o.wg.Wait()
+ fs.Debugf(nil, "Returning error reader")
+ return errorReadCloser{rc}, nil
+ }
+ }
+ }
+ o.wg.Add(1)
+ return wgReadCloser{rc, o.wg}, nil
+}
+
+type errorReadCloser struct {
+ io.ReadCloser
+}
+
+func (rc errorReadCloser) Read(p []byte) (n int, err error) {
+ fs.Debugf(nil, "BOOM: simulated read failure")
+ return 0, errors.New("BOOM: simulated read failure")
+}
+
+type wgReadCloser struct {
+ io.ReadCloser
+ wg *sync.WaitGroup
+}
+
+func (rc wgReadCloser) Close() (err error) {
+ rc.wg.Done()
+ return rc.ReadCloser.Close()
+}
+
+// Make sure aborting the multi-thread copy doesn't overwrite an existing file.
+func TestMultithreadCopyAbort(t *testing.T) {
+ r := fstest.NewRun(t)
+ ctx := context.Background()
+ chunkSize := skipIfNotMultithread(ctx, t, r)
+ size := 2*chunkSize + 1
+
+ if *fstest.SizeLimit > 0 && int64(size) > *fstest.SizeLimit {
+ t.Skipf("exceeded file size limit %d > %d", size, *fstest.SizeLimit)
+ }
+
+ // first write a canary file which we are trying not to overwrite
+ const fileName = "test-multithread-abort"
+ contents := random.String(100)
+ t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
+ canary := r.WriteObject(ctx, fileName, contents, t1)
+ r.CheckRemoteItems(t, canary)
+
+ // Now write a local file to upload
+ file1 := r.WriteFile(fileName, random.String(size), t1)
+ r.CheckLocalItems(t, file1)
+
+ src, err := r.Flocal.NewObject(ctx, fileName)
+ require.NoError(t, err)
+ accounting.GlobalStats().ResetCounters()
+ tr := accounting.GlobalStats().NewTransfer(src, nil)
+
+ defer func() {
+ tr.Done(ctx, err)
+ }()
+ wg := new(sync.WaitGroup)
+ dst, err := multiThreadCopy(ctx, r.Fremote, fileName, errorObject{src, int64(size), wg}, 1, tr)
+ assert.Error(t, err)
+ assert.Nil(t, dst)
+
+ if r.Fremote.Features().PartialUploads {
+ r.CheckRemoteItems(t)
+
+ } else {
+ r.CheckRemoteItems(t, canary)
+ o, err := r.Fremote.NewObject(ctx, fileName)
+ require.NoError(t, err)
+ require.NoError(t, o.Remove(ctx))
+ }
+}
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
new file mode 100644
index 0000000..1ada8f8
--- /dev/null
+++ b/fs/operations/operations.go
@@ -0,0 +1,2742 @@
+// Package operations does generic operations on filesystems and objects
+package operations
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/csv"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "mime"
+ "net/http"
+ "os"
+ "path"
+ "path/filepath"
+ "runtime"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/cache"
+ "github.com/rclone/rclone/fs/config"
+ "github.com/rclone/rclone/fs/filter"
+ "github.com/rclone/rclone/fs/fserrors"
+ "github.com/rclone/rclone/fs/fshttp"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/object"
+ "github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/lib/atexit"
+ "github.com/rclone/rclone/lib/errcount"
+ "github.com/rclone/rclone/lib/pacer"
+ "github.com/rclone/rclone/lib/random"
+ "github.com/rclone/rclone/lib/readers"
+ "github.com/rclone/rclone/lib/transform"
+ "golang.org/x/sync/errgroup"
+ "golang.org/x/text/unicode/norm"
+)
+
+// CheckHashes checks the two files to see if they have common
+// known hash types and compares them
+//
+// Returns.
+//
+// equal - which is equality of the hashes
+//
+// hash - the HashType. This is HashNone if either of the hashes were
+// unset or a compatible hash couldn't be found.
+//
+// err - may return an error which will already have been logged
+//
+// If an error is returned it will return equal as false
+func CheckHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object) (equal bool, ht hash.Type, err error) {
+ common := src.Fs().Hashes().Overlap(dst.Fs().Hashes())
+ // fs.Debugf(nil, "Shared hashes: %v", common)
+ if common.Count() == 0 {
+ return true, hash.None, nil
+ }
+ equal, ht, _, _, err = checkHashes(ctx, src, dst, common.GetOne())
+ return equal, ht, err
+}
+
+var errNoHash = errors.New("no hash available")
+
+// checkHashes does the work of CheckHashes but takes a hash.Type and
+// returns the effective hash type used.
+func checkHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object, ht hash.Type) (equal bool, htOut hash.Type, srcHash, dstHash string, err error) {
+ // Calculate hashes in parallel
+ g, ctx := errgroup.WithContext(ctx)
+ var srcErr, dstErr error
+ g.Go(func() (err error) {
+ srcHash, srcErr = src.Hash(ctx, ht)
+ if srcErr != nil {
+ return srcErr
+ }
+ if srcHash == "" {
+ fs.Debugf(src, "Src hash empty - aborting Dst hash check")
+ return errNoHash
+ }
+ return nil
+ })
+ g.Go(func() (err error) {
+ dstHash, dstErr = dst.Hash(ctx, ht)
+ if dstErr != nil {
+ return dstErr
+ }
+ if dstHash == "" {
+ fs.Debugf(dst, "Dst hash empty - aborting Src hash check")
+ return errNoHash
+ }
+ return nil
+ })
+ err = g.Wait()
+ if err == errNoHash {
+ return true, hash.None, srcHash, dstHash, nil
+ }
+ if srcErr != nil {
+ err = fs.CountError(ctx, srcErr)
+ fs.Errorf(src, "Failed to calculate src hash: %v", err)
+ }
+ if dstErr != nil {
+ err = fs.CountError(ctx, dstErr)
+ fs.Errorf(dst, "Failed to calculate dst hash: %v", err)
+ }
+ if err != nil {
+ return false, ht, srcHash, dstHash, err
+ }
+ if srcHash != dstHash {
+ fs.Debugf(src, "%v = %s (%v)", ht, srcHash, src.Fs())
+ fs.Debugf(dst, "%v = %s (%v)", ht, dstHash, dst.Fs())
+ return false, ht, srcHash, dstHash, nil
+ }
+ fs.Debugf(src, "%v = %s OK", ht, srcHash)
+ return true, ht, srcHash, dstHash, nil
+}
+
+// Equal checks to see if the src and dst objects are equal by looking at
+// size, mtime and hash
+//
+// If the src and dst size are different then it is considered to be
+// not equal. If --size-only is in effect then this is the only check
+// that is done. If --ignore-size is in effect then this check is
+// skipped and the files are considered the same size.
+//
+// If the size is the same and the mtime is the same then it is
+// considered to be equal. This check is skipped if using --checksum.
+//
+// If the size is the same and mtime is different, unreadable or
+// --checksum is set and the hash is the same then the file is
+// considered to be equal. In this case the mtime on the dst is
+// updated if --checksum is not set.
+//
+// Otherwise the file is considered to be not equal including if there
+// were errors reading info.
+func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool {
+ return equal(ctx, src, dst, defaultEqualOpt(ctx))
+}
+
+// DirsEqual is like Equal but for dirs instead of objects.
+// It returns true if two dirs should be considered "equal" for the purposes of syncCopyMove
+// (in other words, true == "skip updating modtime/metadata for this dir".)
+// Unlike Equal, it does not consider size or checksum, as these do not apply to directories.
+func DirsEqual(ctx context.Context, src, dst fs.Directory, opt DirsEqualOpt) (equal bool) {
+ if dst == nil {
+ return false
+ }
+ ci := fs.GetConfig(ctx)
+ if ci.SizeOnly || ci.Immutable || ci.IgnoreExisting || opt.ModifyWindow == fs.ModTimeNotSupported {
+ return true
+ }
+ if ci.IgnoreTimes {
+ return false
+ }
+ if !(opt.SetDirModtime || opt.SetDirMetadata) {
+ return true
+ }
+ srcModTime, dstModTime := src.ModTime(ctx), dst.ModTime(ctx)
+ if srcModTime.IsZero() || dstModTime.IsZero() {
+ return false
+ }
+ dt := dstModTime.Sub(srcModTime)
+ if dt < opt.ModifyWindow && dt > -opt.ModifyWindow {
+ fs.Debugf(dst, "Directory modification time the same (differ by %s, within tolerance %s)", dt, opt.ModifyWindow)
+ return true
+ }
+ if ci.UpdateOlder && dt >= opt.ModifyWindow {
+ fs.Debugf(dst, "Destination directory is newer than source, skipping")
+ return true
+ }
+ return false
+}
+
+// sizeDiffers compare the size of src and dst taking into account the
+// various ways of ignoring sizes
+func sizeDiffers(ctx context.Context, src, dst fs.ObjectInfo) bool {
+ ci := fs.GetConfig(ctx)
+ if ci.IgnoreSize || src.Size() < 0 || dst.Size() < 0 {
+ return false
+ }
+ if src.Size() == dst.Size() {
+ fs.Debugf(dst, "size = %d OK", dst.Size())
+ return false
+ }
+ fs.Debugf(src, "size = %d (%v)", src.Size(), src.Fs())
+ fs.Debugf(dst, "size = %d (%v)", dst.Size(), dst.Fs())
+ return true
+}
+
+var checksumWarning sync.Once
+
+// options for equal function()
+type equalOpt struct {
+ sizeOnly bool // if set only check size
+ checkSum bool // if set check checksum+size instead of modtime+size
+ updateModTime bool // if set update the modtime if hashes identical and checking with modtime+size
+ forceModTimeMatch bool // if set assume modtimes match
+}
+
+// default set of options for equal()
+func defaultEqualOpt(ctx context.Context) equalOpt {
+ ci := fs.GetConfig(ctx)
+ return equalOpt{
+ sizeOnly: ci.SizeOnly,
+ checkSum: ci.CheckSum,
+ updateModTime: !ci.NoUpdateModTime,
+ forceModTimeMatch: false,
+ }
+}
+
+// DirsEqualOpt represents options for DirsEqual function()
+type DirsEqualOpt struct {
+ ModifyWindow time.Duration // Max time diff to be considered the same
+ SetDirModtime bool // whether to consider dir modtime
+ SetDirMetadata bool // whether to consider dir metadata
+}
+
+var modTimeUploadOnce sync.Once
+
+// emit a log if we are about to upload a file to set its modification time
+func logModTimeUpload(dst fs.Object) {
+ modTimeUploadOnce.Do(func() {
+ fs.Logf(dst.Fs(), "Forced to upload files to set modification times on this backend.")
+ })
+}
+
+// EqualFn allows replacing Equal() with a custom function during NeedTransfer()
+type (
+ EqualFn func(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool
+ equalFnContextKey struct{}
+)
+
+var equalFnKey = equalFnContextKey{}
+
+// WithEqualFn stores equalFn in ctx and returns a copy of ctx in which equalFnKey = equalFn
+func WithEqualFn(ctx context.Context, equalFn EqualFn) context.Context {
+ return context.WithValue(ctx, equalFnKey, equalFn)
+}
+
+func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt) bool {
+ ci := fs.GetConfig(ctx)
+ logger, _ := GetLogger(ctx)
+ if sizeDiffers(ctx, src, dst) {
+ fs.Debug(src, "Sizes differ")
+ logger(ctx, Differ, src, dst, nil)
+ return false
+ }
+ if opt.sizeOnly {
+ fs.Debugf(src, "Sizes identical")
+ logger(ctx, Match, src, dst, nil)
+ return true
+ }
+
+ // Assert: Size is equal or being ignored
+
+ // If checking checksum and not modtime
+ if opt.checkSum {
+ // Check the hash
+ same, ht, _ := CheckHashes(ctx, src, dst)
+ if !same {
+ fs.Debugf(src, "%v differ", ht)
+ logger(ctx, Differ, src, dst, nil)
+ return false
+ }
+ if ht == hash.None {
+ common := src.Fs().Hashes().Overlap(dst.Fs().Hashes())
+ if common.Count() == 0 {
+ checksumWarning.Do(func() {
+ fs.Logf(dst.Fs(), "--checksum is in use but the source and destination have no hashes in common; falling back to --size-only")
+ })
+ }
+ fs.Debugf(src, "Size of src and dst objects identical")
+ } else {
+ fs.Debugf(src, "Size and %v of src and dst objects identical", ht)
+ }
+ logger(ctx, Match, src, dst, nil)
+ return true
+ }
+
+ srcModTime := src.ModTime(ctx)
+ if !opt.forceModTimeMatch {
+ // Sizes the same so check the mtime
+ modifyWindow := fs.GetModifyWindow(ctx, src.Fs(), dst.Fs())
+ if modifyWindow == fs.ModTimeNotSupported {
+ fs.Debugf(src, "Sizes identical")
+ logger(ctx, Match, src, dst, nil)
+ return true
+ }
+ dstModTime := dst.ModTime(ctx)
+ dt := dstModTime.Sub(srcModTime)
+ if dt < modifyWindow && dt > -modifyWindow {
+ fs.Debugf(src, "Size and modification time the same (differ by %s, within tolerance %s)", dt, modifyWindow)
+ logger(ctx, Match, src, dst, nil)
+ return true
+ }
+
+ fs.Debugf(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime)
+ }
+
+ // Check if the hashes are the same
+ same, ht, _ := CheckHashes(ctx, src, dst)
+ if !same {
+ fs.Debugf(src, "%v differ", ht)
+ logger(ctx, Differ, src, dst, nil)
+ return false
+ }
+ if ht == hash.None && !ci.RefreshTimes {
+ // if couldn't check hash, return that they differ
+ logger(ctx, Differ, src, dst, nil)
+ return false
+ }
+
+ // mod time differs but hash is the same to reset mod time if required
+ if opt.updateModTime {
+ if !SkipDestructive(ctx, src, "update modification time") {
+ // Size and hash the same but mtime different
+ // Error if objects are treated as immutable
+ if ci.Immutable {
+ fs.Errorf(dst, "Timestamp mismatch between immutable objects")
+ logger(ctx, Differ, src, dst, nil)
+ return false
+ }
+ // Update the mtime of the dst object here
+ err := dst.SetModTime(ctx, srcModTime)
+ if errors.Is(err, fs.ErrorCantSetModTime) {
+ logModTimeUpload(dst)
+ fs.Infof(dst, "src and dst identical but can't set mod time without re-uploading")
+ logger(ctx, Differ, src, dst, nil)
+ return false
+ } else if errors.Is(err, fs.ErrorCantSetModTimeWithoutDelete) {
+ logModTimeUpload(dst)
+ fs.Infof(dst, "src and dst identical but can't set mod time without deleting and re-uploading")
+ // Remove the file if BackupDir isn't set. If BackupDir is set we would rather have the old file
+ // put in the BackupDir than deleted which is what will happen if we don't delete it.
+ if ci.BackupDir == "" {
+ err = dst.Remove(ctx)
+ if err != nil {
+ fs.Errorf(dst, "failed to delete before re-upload: %v", err)
+ }
+ }
+ logger(ctx, Differ, src, dst, nil)
+ return false
+ } else if err != nil {
+ err = fs.CountError(ctx, err)
+ fs.Errorf(dst, "Failed to set modification time: %v", err)
+ } else {
+ fs.Infof(src, "Updated modification time in destination")
+ }
+ }
+ }
+ logger(ctx, Match, src, dst, nil)
+ return true
+}
+
+// CommonHash returns a single hash.Type and a HashOption with that
+// type which is in common between the two fs.Fs.
+func CommonHash(ctx context.Context, fa, fb fs.Info) (hash.Type, *fs.HashesOption) {
+ ci := fs.GetConfig(ctx)
+ // work out which hash to use - limit to 1 hash in common
+ var common hash.Set
+ hashType := hash.None
+ if !ci.IgnoreChecksum {
+ common = fb.Hashes().Overlap(fa.Hashes())
+ if common.Count() > 0 {
+ hashType = common.GetOne()
+ common = hash.Set(hashType)
+ }
+ }
+ return hashType, &fs.HashesOption{Hashes: common}
+}
+
+// SameObject returns true if src and dst could be pointing to the
+// same object.
+func SameObject(src, dst fs.Object) bool {
+ srcFs, dstFs := src.Fs(), dst.Fs()
+ if !SameConfig(srcFs, dstFs) {
+ // If same remote type then check ID of objects if available
+ doSrcID, srcIDOK := src.(fs.IDer)
+ doDstID, dstIDOK := dst.(fs.IDer)
+ if srcIDOK && dstIDOK && SameRemoteType(srcFs, dstFs) {
+ srcID, dstID := doSrcID.ID(), doDstID.ID()
+ if srcID != "" && srcID == dstID {
+ return true
+ }
+ }
+ return false
+ }
+ srcPath := path.Join(srcFs.Root(), src.Remote())
+ dstPath := path.Join(dstFs.Root(), dst.Remote())
+ if srcFs.Features().IsLocal && dstFs.Features().IsLocal && runtime.GOOS == "darwin" {
+ if norm.NFC.String(srcPath) == norm.NFC.String(dstPath) {
+ return true
+ }
+ }
+ if dst.Fs().Features().CaseInsensitive {
+ srcPath = strings.ToLower(srcPath)
+ dstPath = strings.ToLower(dstPath)
+ }
+ return srcPath == dstPath
+}
+
+// Move src object to dst or fdst if nil. If dst is nil then it uses
+// remote as the name of the new object.
+//
+// Note that you must check the destination does not exist before
+// calling this and pass it as dst. If you pass dst=nil and the
+// destination does exist then this may create duplicates or return
+// errors.
+//
+// It returns the destination object if possible. Note that this may
+// be nil.
+//
+// This is accounted as a check.
+func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
+ return move(ctx, fdst, dst, remote, src, false)
+}
+
+// MoveTransfer moves src object to dst or fdst if nil. If dst is nil
+// then it uses remote as the name of the new object.
+//
+// This is identical to Move but is accounted as a transfer.
+func MoveTransfer(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
+ return move(ctx, fdst, dst, remote, src, true)
+}
+
+// move - see Move for help
+func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object, isTransfer bool) (newDst fs.Object, err error) {
+ origRemote := remote // avoid double-transform on fallback to copy
+ remote = transform.Path(ctx, remote, false)
+ ci := fs.GetConfig(ctx)
+ newDst = dst
+ if ci.DryRun && dst != nil && SameObject(src, dst) && src.Remote() == transform.Path(ctx, dst.Remote(), false) {
+ return // avoid SkipDestructive log for objects that won't really be moved
+ }
+ var tr *accounting.Transfer
+ if isTransfer {
+ tr = accounting.Stats(ctx).NewTransfer(src, fdst)
+ } else {
+ tr = accounting.Stats(ctx).NewCheckingTransfer(src, "moving")
+ }
+ defer func() {
+ if err == nil {
+ accounting.Stats(ctx).Renames(1)
+ }
+ tr.Done(ctx, err)
+ }()
+ action := "move"
+ if remote != src.Remote() {
+ action += " to " + remote
+ }
+ if SkipDestructive(ctx, src, action) {
+ in := tr.Account(ctx, nil)
+ in.DryRun(src.Size())
+ return newDst, nil
+ }
+ // See if we have Move available
+ if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && (fdst.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) {
+ // Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive)
+ if dst != nil {
+ remote = transform.Path(ctx, dst.Remote(), false)
+ if !SameObject(src, dst) {
+ err = DeleteFile(ctx, dst)
+ if err != nil {
+ return newDst, err
+ }
+ } else if src.Remote() == remote {
+ return newDst, nil
+ } else if needsMoveCaseInsensitive(fdst, fdst, remote, src.Remote(), false) {
+ doMove = func(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+ return MoveCaseInsensitive(ctx, fdst, fdst, remote, src.Remote(), false, src)
+ }
+ }
+ } else if needsMoveCaseInsensitive(fdst, fdst, remote, src.Remote(), false) {
+ doMove = func(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+ return MoveCaseInsensitive(ctx, fdst, fdst, remote, src.Remote(), false, src)
+ }
+ }
+ // Move dst <- src
+ in := tr.Account(ctx, nil) // account the transfer
+ in.ServerSideTransferStart()
+ newDst, err = doMove(ctx, src, remote)
+ switch err {
+ case nil:
+ if newDst != nil && src.String() != newDst.String() {
+ fs.Infof(src, "Moved (server-side) to: %s", newDst.String())
+ } else {
+ fs.Infof(src, "Moved (server-side)")
+ }
+ in.ServerSideMoveEnd(newDst.Size()) // account the bytes for the server-side transfer
+ _ = in.Close()
+ return newDst, nil
+ case fs.ErrorCantMove:
+ fs.Debugf(src, "Can't move, switching to copy")
+ _ = in.Close()
+ default:
+ err = fs.CountError(ctx, err)
+ fs.Errorf(src, "Couldn't move: %v", err)
+ _ = in.Close()
+ return newDst, err
+ }
+ }
+ // Move not found or didn't work so copy dst <- src
+ if origRemote != remote {
+ dst = nil
+ }
+ newDst, err = Copy(ctx, fdst, dst, origRemote, src)
+ if err != nil {
+ fs.Errorf(src, "Not deleting source as copy failed: %v", err)
+ return newDst, err
+ }
+ // Delete src if no error on copy
+ return newDst, DeleteFile(ctx, src)
+}
+
+// CanServerSideMove returns true if fdst support server-side moves or
+// server-side copies
+//
+// Some remotes simulate rename by server-side copy and delete, so include
+// remotes that implements either Mover or Copier.
+func CanServerSideMove(fdst fs.Fs) bool {
+ canMove := fdst.Features().Move != nil
+ canCopy := fdst.Features().Copy != nil
+ return canMove || canCopy
+}
+
+// SuffixName adds the current --suffix to the remote, obeying
+// --suffix-keep-extension if set
+func SuffixName(ctx context.Context, remote string) string {
+ ci := fs.GetConfig(ctx)
+ if ci.Suffix == "" {
+ return remote
+ }
+ if ci.SuffixKeepExtension {
+ return transform.SuffixKeepExtension(remote, ci.Suffix)
+ }
+ return remote + ci.Suffix
+}
+
+// DeleteFileWithBackupDir deletes a single file respecting --dry-run
+// and accumulating stats and errors.
+//
+// If backupDir is set then it moves the file to there instead of
+// deleting
+func DeleteFileWithBackupDir(ctx context.Context, dst fs.Object, backupDir fs.Fs) (err error) {
+ tr := accounting.Stats(ctx).NewCheckingTransfer(dst, "deleting")
+ defer func() {
+ tr.Done(ctx, err)
+ }()
+ err = accounting.Stats(ctx).DeleteFile(ctx, dst.Size())
+ if err != nil {
+ return err
+ }
+ action, actioned := "delete", "Deleted"
+ if backupDir != nil {
+ action, actioned = "move into backup dir", "Moved into backup dir"
+ }
+ skip := SkipDestructive(ctx, dst, action)
+ if skip {
+ // do nothing
+ } else if backupDir != nil {
+ err = MoveBackupDir(ctx, backupDir, dst)
+ } else {
+ err = dst.Remove(ctx)
+ }
+ if err != nil {
+ fs.Errorf(dst, "Couldn't %s: %v", action, err)
+ err = fs.CountError(ctx, err)
+ } else if !skip {
+ fs.Infof(dst, "%s", actioned)
+ }
+ return err
+}
+
+// DeleteFile deletes a single file respecting --dry-run and accumulating stats and errors.
+//
+// If useBackupDir is set and --backup-dir is in effect then it moves
+// the file to there instead of deleting
+func DeleteFile(ctx context.Context, dst fs.Object) (err error) {
+ return DeleteFileWithBackupDir(ctx, dst, nil)
+}
+
+// DeleteFilesWithBackupDir removes all the files passed in the
+// channel
+//
+// If backupDir is set the files will be placed into that directory
+// instead of being deleted.
+func DeleteFilesWithBackupDir(ctx context.Context, toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error {
+ var wg sync.WaitGroup
+ ci := fs.GetConfig(ctx)
+ wg.Add(ci.Checkers)
+ var errorCount atomic.Int32
+ var fatalErrorCount atomic.Int32
+
+ for range ci.Checkers {
+ go func() {
+ defer wg.Done()
+ for dst := range toBeDeleted {
+ err := DeleteFileWithBackupDir(ctx, dst, backupDir)
+ if err != nil {
+ errorCount.Add(1)
+ logger, _ := GetLogger(ctx)
+ logger(ctx, TransferError, nil, dst, err)
+ if fserrors.IsFatalError(err) {
+ fs.Errorf(dst, "Got fatal error on delete: %s", err)
+ fatalErrorCount.Add(1)
+ return
+ }
+ }
+ }
+ }()
+ }
+ fs.Debugf(nil, "Waiting for deletions to finish")
+ wg.Wait()
+ if errorCount.Load() > 0 {
+ err := fmt.Errorf("failed to delete %d files", errorCount.Load())
+ if fatalErrorCount.Load() > 0 {
+ return fserrors.FatalError(err)
+ }
+ return err
+ }
+ return nil
+}
+
+// DeleteFiles removes all the files passed in the channel
+func DeleteFiles(ctx context.Context, toBeDeleted fs.ObjectsChan) error {
+ return DeleteFilesWithBackupDir(ctx, toBeDeleted, nil)
+}
+
+// ReadFile reads the object into memory and accounts it
+func ReadFile(ctx context.Context, o fs.Object) (b []byte, err error) {
+ tr := accounting.Stats(ctx).NewTransfer(o, nil)
+ defer func() {
+ tr.Done(ctx, err)
+ }()
+ in0, err := Open(ctx, o)
+ if err != nil {
+ return nil, fmt.Errorf("failed to open %v: %w", o, err)
+ }
+ in := tr.Account(ctx, in0).WithBuffer() // account and buffer the transfer
+ defer fs.CheckClose(in, &err) // closes in0 also
+ b, err = io.ReadAll(in)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read %v: %w", o, err)
+ }
+ return b, nil
+}
+
+// SameRemoteType returns true if fdst and fsrc are the same type
+func SameRemoteType(fdst, fsrc fs.Info) bool {
+ return fmt.Sprintf("%T", fdst) == fmt.Sprintf("%T", fsrc)
+}
+
+// SameConfig returns true if fdst and fsrc are using the same config
+// file entry
+func SameConfig(fdst, fsrc fs.Info) bool {
+ return fdst.Name() == fsrc.Name()
+}
+
+// SameConfigArr returns true if any of []fsrcs has same config file entry with fdst
+func SameConfigArr(fdst fs.Info, fsrcs []fs.Fs) bool {
+ for _, fsrc := range fsrcs {
+ if fdst.Name() == fsrc.Name() {
+ return true
+ }
+ }
+ return false
+}
+
+// Same returns true if fdst and fsrc point to the same underlying Fs
+func Same(fdst, fsrc fs.Info) bool {
+ return SameConfig(fdst, fsrc) && strings.Trim(fdst.Root(), "/") == strings.Trim(fsrc.Root(), "/")
+}
+
+// fixRoot returns the Root with a trailing / if not empty.
+//
+// It returns a case folded version for case insensitive file systems
+func fixRoot(f fs.Info) (s string, folded string) {
+ s = strings.Trim(filepath.ToSlash(f.Root()), "/")
+ if s != "" {
+ s += "/"
+ }
+ folded = s
+ if f.Features().CaseInsensitive {
+ folded = strings.ToLower(s)
+ }
+ return s, folded
+}
+
+// OverlappingFilterCheck returns true if fdst and fsrc point to the same
+// underlying Fs and they overlap without fdst being excluded by any filter rule.
+func OverlappingFilterCheck(ctx context.Context, fdst fs.Fs, fsrc fs.Fs) bool {
+ if !SameConfig(fdst, fsrc) {
+ return false
+ }
+ fdstRoot, fdstRootFolded := fixRoot(fdst)
+ fsrcRoot, fsrcRootFolded := fixRoot(fsrc)
+ if fdstRootFolded == fsrcRootFolded {
+ return true
+ } else if strings.HasPrefix(fdstRootFolded, fsrcRootFolded) {
+ fdstRelative := fdstRoot[len(fsrcRoot):]
+ return filterCheck(ctx, fsrc, fdstRelative)
+ } else if strings.HasPrefix(fsrcRootFolded, fdstRootFolded) {
+ fsrcRelative := fsrcRoot[len(fdstRoot):]
+ return filterCheck(ctx, fdst, fsrcRelative)
+ }
+ return false
+}
+
+// filterCheck checks if dir is included in f
+func filterCheck(ctx context.Context, f fs.Fs, dir string) bool {
+ fi := filter.GetConfig(ctx)
+ includeDirectory := fi.IncludeDirectory(ctx, f)
+ include, err := includeDirectory(dir)
+ if err != nil {
+ fs.Errorf(f, "Failed to discover whether directory is included: %v", err)
+ return true
+ }
+ return include
+}
+
+// SameDir returns true if fdst and fsrc point to the same
+// underlying Fs and they are the same directory.
+func SameDir(fdst, fsrc fs.Info) bool {
+ if !SameConfig(fdst, fsrc) {
+ return false
+ }
+ _, fdstRootFolded := fixRoot(fdst)
+ _, fsrcRootFolded := fixRoot(fsrc)
+ return fdstRootFolded == fsrcRootFolded
+}
+
+// Retry runs fn up to maxTries times if it returns a retriable error
+func Retry(ctx context.Context, o any, maxTries int, fn func() error) (err error) {
+ for tries := 1; tries <= maxTries; tries++ {
+ // Call the function which might error
+ err = fn()
+ if err == nil {
+ break
+ }
+ // End if ctx is in error
+ if fserrors.ContextError(ctx, &err) {
+ break
+ }
+ // Retry if err returned a retry error
+ if fserrors.IsRetryError(err) || fserrors.ShouldRetry(err) {
+ fs.Debugf(o, "Received error: %v - low level retry %d/%d", err, tries, maxTries)
+ continue
+ } else if t, ok := pacer.IsRetryAfter(err); ok {
+ fs.Debugf(o, "Sleeping for %v (as indicated by the server) to obey Retry-After error: %v", t, err)
+ time.Sleep(t)
+ continue
+ }
+ break
+ }
+ return err
+}
+
+// ListFn lists the Fs to the supplied function
+//
+// Lists in parallel which may get them out of order
+func ListFn(ctx context.Context, f fs.Fs, fn func(fs.Object)) error {
+ ci := fs.GetConfig(ctx)
+ return walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
+ entries.ForObject(fn)
+ return nil
+ })
+}
+
+// StdoutMutex mutex for synchronized output on stdout
+var StdoutMutex sync.Mutex
+
+// SyncPrintf is a global var holding the Printf function so that it
+// can be overridden.
+//
+// This writes to stdout holding the StdoutMutex. If you are going to
+// override it and write to os.Stdout then you should hold the
+// StdoutMutex too.
+var SyncPrintf = func(format string, a ...any) {
+ StdoutMutex.Lock()
+ defer StdoutMutex.Unlock()
+ fmt.Printf(format, a...)
+}
+
+// SyncFprintf - Synchronized fmt.Fprintf
+//
+// Ignores errors from Fprintf.
+//
+// Prints to stdout if w is nil
+func SyncFprintf(w io.Writer, format string, a ...any) {
+ if w == nil || w == os.Stdout {
+ SyncPrintf(format, a...)
+ } else {
+ StdoutMutex.Lock()
+ defer StdoutMutex.Unlock()
+ _, _ = fmt.Fprintf(w, format, a...)
+ }
+}
+
+// SizeString make string representation of size for output
+//
+// Optional human-readable format including a binary suffix
+func SizeString(size int64, humanReadable bool) string {
+ if humanReadable {
+ if size < 0 {
+ return "-" + fs.SizeSuffix(-size).String()
+ }
+ return fs.SizeSuffix(size).String()
+ }
+ return strconv.FormatInt(size, 10)
+}
+
+// SizeStringField make string representation of size for output in fixed width field
+//
+// Optional human-readable format including a binary suffix
+// Argument rawWidth is used to format field with of raw value. When humanReadable
+// option the width is hard coded to 9, since SizeSuffix strings have precision 3
+// and longest value will be "999.999Ei". This way the width can be optimized
+// depending to the humanReadable option. To always use a longer width the return
+// value can always be fed into another format string with a specific field with.
+func SizeStringField(size int64, humanReadable bool, rawWidth int) string {
+ str := SizeString(size, humanReadable)
+ if humanReadable {
+ return fmt.Sprintf("%9s", str)
+ }
+ return fmt.Sprintf("%[2]*[1]s", str, rawWidth)
+}
+
+// CountString make string representation of count for output
+//
+// Optional human-readable format including a decimal suffix
+func CountString(count int64, humanReadable bool) string {
+ if humanReadable {
+ if count < 0 {
+ return "-" + fs.CountSuffix(-count).String()
+ }
+ return fs.CountSuffix(count).String()
+ }
+ return strconv.FormatInt(count, 10)
+}
+
+// CountStringField make string representation of count for output in fixed width field
+//
+// Similar to SizeStringField, but human readable with decimal prefix and field width 8
+// since there is no 'i' in the decimal prefix symbols (e.g. "999.999E")
+func CountStringField(count int64, humanReadable bool, rawWidth int) string {
+ str := CountString(count, humanReadable)
+ if humanReadable {
+ return fmt.Sprintf("%8s", str)
+ }
+ return fmt.Sprintf("%[2]*[1]s", str, rawWidth)
+}
+
+// List the Fs to the supplied writer
+//
+// Shows size and path - obeys includes and excludes.
+//
+// Lists in parallel which may get them out of order
+func List(ctx context.Context, f fs.Fs, w io.Writer) error {
+ ci := fs.GetConfig(ctx)
+ return ListFn(ctx, f, func(o fs.Object) {
+ SyncFprintf(w, "%s %s\n", SizeStringField(o.Size(), ci.HumanReadable, 9), o.Remote())
+ })
+}
+
+// ListLong lists the Fs to the supplied writer
+//
+// Shows size, mod time and path - obeys includes and excludes.
+//
+// Lists in parallel which may get them out of order
+func ListLong(ctx context.Context, f fs.Fs, w io.Writer) error {
+ ci := fs.GetConfig(ctx)
+ return ListFn(ctx, f, func(o fs.Object) {
+ tr := accounting.Stats(ctx).NewCheckingTransfer(o, "listing")
+ defer func() {
+ tr.Done(ctx, nil)
+ }()
+ modTime := o.ModTime(ctx)
+ SyncFprintf(w, "%s %s %s\n", SizeStringField(o.Size(), ci.HumanReadable, 9), modTime.Local().Format("2006-01-02 15:04:05.000000000"), o.Remote())
+ })
+}
+
+// HashSum returns the human-readable hash for ht passed in. This may
+// be UNSUPPORTED or ERROR. If it isn't returning a valid hash it will
+// return an error.
+func HashSum(ctx context.Context, ht hash.Type, base64Encoded bool, downloadFlag bool, o fs.Object) (string, error) {
+ var sum string
+ var err error
+
+ // If downloadFlag is true, download and hash the file.
+ // If downloadFlag is false, call o.Hash asking the remote for the hash
+ if downloadFlag {
+ // Setup: Define accounting, open the file with NewReOpen to provide restarts, account for the transfer, and setup a multi-hasher with the appropriate type
+ // Execution: io.Copy file to hasher, get hash and encode in hex
+
+ tr := accounting.Stats(ctx).NewTransfer(o, nil)
+ defer func() {
+ tr.Done(ctx, err)
+ }()
+
+ // Open with NewReOpen to provide restarts
+ var options []fs.OpenOption
+ for _, option := range fs.GetConfig(ctx).DownloadHeaders {
+ options = append(options, option)
+ }
+ var in io.ReadCloser
+ in, err = Open(ctx, o, options...)
+ if err != nil {
+ return "ERROR", fmt.Errorf("failed to open file %v: %w", o, err)
+ }
+
+ // Account and buffer the transfer
+ in = tr.Account(ctx, in).WithBuffer()
+
+ // Setup hasher
+ hasher, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht))
+ if err != nil {
+ return "UNSUPPORTED", fmt.Errorf("hash unsupported: %w", err)
+ }
+
+ // Copy to hasher, downloading the file and passing directly to hash
+ _, err = io.Copy(hasher, in)
+ if err != nil {
+ return "ERROR", fmt.Errorf("failed to copy file to hasher: %w", err)
+ }
+
+ // Get hash as hex or base64 encoded string
+ sum, err = hasher.SumString(ht, base64Encoded)
+ if err != nil {
+ return "ERROR", fmt.Errorf("hasher returned an error: %w", err)
+ }
+ } else {
+ tr := accounting.Stats(ctx).NewCheckingTransfer(o, "hashing")
+ defer func() {
+ tr.Done(ctx, err)
+ }()
+
+ sum, err = o.Hash(ctx, ht)
+ if base64Encoded {
+ hexBytes, _ := hex.DecodeString(sum)
+ sum = base64.URLEncoding.EncodeToString(hexBytes)
+ }
+ if err == hash.ErrUnsupported {
+ return "", fmt.Errorf("hash unsupported: %w", err)
+ }
+ if err != nil {
+ return "", fmt.Errorf("failed to get hash %v from backend: %w", ht, err)
+ }
+ }
+
+ return sum, nil
+}
+
+// HashLister does an md5sum equivalent for the hash type passed in
+// Updated to handle both standard hex encoding and base64
+// Updated to perform multiple hashes concurrently
+func HashLister(ctx context.Context, ht hash.Type, outputBase64 bool, downloadFlag bool, f fs.Fs, w io.Writer) error {
+ width := hash.Width(ht, outputBase64)
+ // Use --checkers concurrency unless downloading in which case use --transfers
+ concurrency := fs.GetConfig(ctx).Checkers
+ if downloadFlag {
+ concurrency = fs.GetConfig(ctx).Transfers
+ }
+ concurrencyControl := make(chan struct{}, concurrency)
+ var wg sync.WaitGroup
+ err := ListFn(ctx, f, func(o fs.Object) {
+ wg.Add(1)
+ concurrencyControl <- struct{}{}
+ go func() {
+ defer func() {
+ <-concurrencyControl
+ wg.Done()
+ }()
+ sum, err := HashSum(ctx, ht, outputBase64, downloadFlag, o)
+ if err != nil {
+ fs.Errorf(o, "%v", fs.CountError(ctx, err))
+ return
+ }
+ SyncFprintf(w, "%*s %s\n", width, sum, o.Remote())
+ }()
+ })
+ wg.Wait()
+ return err
+}
+
+// HashSumStream outputs a line compatible with md5sum to w based on the
+// input stream in and the hash type ht passed in. If outputBase64 is
+// set then the hash will be base64 instead of hexadecimal.
+func HashSumStream(ht hash.Type, outputBase64 bool, in io.ReadCloser, w io.Writer) error {
+ hasher, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht))
+ if err != nil {
+ return fmt.Errorf("hash unsupported: %w", err)
+ }
+ written, err := io.Copy(hasher, in)
+ fs.Debugf(nil, "Creating %s hash of %d bytes read from input stream", ht, written)
+ if err != nil {
+ return fmt.Errorf("failed to copy input to hasher: %w", err)
+ }
+ sum, err := hasher.SumString(ht, outputBase64)
+ if err != nil {
+ return fmt.Errorf("hasher returned an error: %w", err)
+ }
+ width := hash.Width(ht, outputBase64)
+ SyncFprintf(w, "%*s -\n", width, sum)
+ return nil
+}
+
+// Count counts the objects and their sizes in the Fs
+//
+// Obeys includes and excludes
+func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, sizelessObjects int64, err error) {
+ err = ListFn(ctx, f, func(o fs.Object) {
+ atomic.AddInt64(&objects, 1)
+ objectSize := o.Size()
+ if objectSize < 0 {
+ atomic.AddInt64(&sizelessObjects, 1)
+ } else if objectSize > 0 {
+ atomic.AddInt64(&size, objectSize)
+ }
+ })
+ return
+}
+
+// ConfigMaxDepth returns the depth to use for a recursive or non recursive listing.
+func ConfigMaxDepth(ctx context.Context, recursive bool) int {
+ ci := fs.GetConfig(ctx)
+ depth := ci.MaxDepth
+ if !recursive && depth < 0 {
+ depth = 1
+ }
+ return depth
+}
+
+// ListDir lists the directories/buckets/containers in the Fs to the supplied writer
+func ListDir(ctx context.Context, f fs.Fs, w io.Writer) error {
+ ci := fs.GetConfig(ctx)
+ return walk.ListR(ctx, f, "", false, ConfigMaxDepth(ctx, false), walk.ListDirs, func(entries fs.DirEntries) error {
+ entries.ForDir(func(dir fs.Directory) {
+ if dir != nil {
+ SyncFprintf(w, "%s %13s %s %s\n", SizeStringField(dir.Size(), ci.HumanReadable, 12), dir.ModTime(ctx).Local().Format("2006-01-02 15:04:05"), CountStringField(dir.Items(), ci.HumanReadable, 9), dir.Remote())
+ }
+ })
+ return nil
+ })
+}
+
+// Mkdir makes a destination directory or container
+func Mkdir(ctx context.Context, f fs.Fs, dir string) error {
+ if SkipDestructive(ctx, fs.LogDirName(f, dir), "make directory") {
+ return nil
+ }
+ fs.Infof(fs.LogDirName(f, dir), "Making directory")
+ err := f.Mkdir(ctx, dir)
+ if err != nil {
+ err = fs.CountError(ctx, err)
+ return err
+ }
+ return nil
+}
+
+// MkdirMetadata makes a destination directory or container with metadata
+//
+// If the destination Fs doesn't support this it will fall back to
+// Mkdir and in this case newDst will be nil.
+func MkdirMetadata(ctx context.Context, f fs.Fs, dir string, metadata fs.Metadata) (newDst fs.Directory, err error) {
+ do := f.Features().MkdirMetadata
+ if do == nil {
+ return nil, Mkdir(ctx, f, dir)
+ }
+ logName := fs.LogDirName(f, dir)
+ if SkipDestructive(ctx, logName, "make directory") {
+ return nil, nil
+ }
+ fs.Debugf(fs.LogDirName(f, dir), "Making directory with metadata")
+ newDst, err = do(ctx, dir, metadata)
+ if err != nil {
+ err = fs.CountError(ctx, err)
+ return nil, err
+ }
+ if mtime, ok := metadata["mtime"]; ok {
+ fs.Infof(logName, "Made directory with metadata (mtime=%s)", mtime)
+ } else {
+ fs.Infof(logName, "Made directory with metadata")
+ }
+ return newDst, err
+}
+
+// MkdirModTime makes a destination directory or container with modtime
+//
+// It will try to make the directory with MkdirMetadata and if that
+// succeeds it will return a non-nil newDst. In all other cases newDst
+// will be nil.
+//
+// If the directory was created with MkDir then it will attempt to use
+// Fs.DirSetModTime to update the directory modtime if available.
+func MkdirModTime(ctx context.Context, f fs.Fs, dir string, modTime time.Time) (newDst fs.Directory, err error) {
+ logName := fs.LogDirName(f, dir)
+ if SkipDestructive(ctx, logName, "make directory") {
+ return nil, nil
+ }
+ metadata := fs.Metadata{
+ "mtime": modTime.Format(time.RFC3339Nano),
+ }
+ newDst, err = MkdirMetadata(ctx, f, dir, metadata)
+ if err != nil {
+ return nil, err
+ }
+ if newDst != nil {
+ // The directory was created and we have logged already
+ return newDst, nil
+ }
+ // The directory was created with Mkdir then we should try to set the time
+ if do := f.Features().DirSetModTime; do != nil {
+ err = do(ctx, dir, modTime)
+ }
+ fs.Infof(logName, "Made directory with modification time %v", modTime)
+ return newDst, err
+}
+
+// TryRmdir removes a container but not if not empty. It doesn't
+// count errors but may return one.
+func TryRmdir(ctx context.Context, f fs.Fs, dir string) error {
+ accounting.Stats(ctx).DeletedDirs(1)
+ if SkipDestructive(ctx, fs.LogDirName(f, dir), "remove directory") {
+ return nil
+ }
+ fs.Infof(fs.LogDirName(f, dir), "Removing directory")
+ return f.Rmdir(ctx, dir)
+}
+
+// Rmdir removes a container but not if not empty
+func Rmdir(ctx context.Context, f fs.Fs, dir string) error {
+ err := TryRmdir(ctx, f, dir)
+ if err != nil {
+ err = fs.CountError(ctx, err)
+ return err
+ }
+ return err
+}
+
+// Purge removes a directory and all of its contents
+func Purge(ctx context.Context, f fs.Fs, dir string) (err error) {
+ doFallbackPurge := true
+ if doPurge := f.Features().Purge; doPurge != nil {
+ doFallbackPurge = false
+ accounting.Stats(ctx).DeletedDirs(1)
+ if SkipDestructive(ctx, fs.LogDirName(f, dir), "purge directory") {
+ return nil
+ }
+ err = doPurge(ctx, dir)
+ if errors.Is(err, fs.ErrorCantPurge) {
+ doFallbackPurge = true
+ }
+ }
+ if doFallbackPurge {
+ // DeleteFiles and Rmdir observe --dry-run
+ err = DeleteFiles(ctx, listToChan(ctx, f, dir))
+ if err != nil {
+ return err
+ }
+ err = Rmdirs(ctx, f, dir, false)
+ }
+ if err != nil {
+ err = fs.CountError(ctx, err)
+ return err
+ }
+ return nil
+}
+
+// Delete removes all the contents of a container. Unlike Purge, it
+// obeys includes and excludes.
+func Delete(ctx context.Context, f fs.Fs) error {
+ ci := fs.GetConfig(ctx)
+ delChan := make(fs.ObjectsChan, ci.Checkers)
+ delErr := make(chan error, 1)
+ go func() {
+ delErr <- DeleteFiles(ctx, delChan)
+ }()
+ err := ListFn(ctx, f, func(o fs.Object) {
+ delChan <- o
+ })
+ close(delChan)
+ delError := <-delErr
+ if err == nil {
+ err = delError
+ }
+ return err
+}
+
+// RemoveExisting removes an existing file in a safe way so that it
+// can be restored if the operation fails.
+//
+// This first detects if there is an existing file and renames it to a
+// temporary name if there is.
+//
+// The returned cleanup function should be called on a defer statement
+// with a pointer to the error returned. It will revert the changes if
+// there is an error or delete the existing file if not.
+func RemoveExisting(ctx context.Context, f fs.Fs, remote string, operation string) (cleanup func(*error), err error) {
+ existingObj, err := f.NewObject(ctx, remote)
+ if err != nil {
+ return func(*error) {}, nil
+ }
+ doMove := f.Features().Move
+ if doMove == nil {
+ return nil, fmt.Errorf("%s: destination file exists already and can't rename", operation)
+ }
+
+ // Avoid making the leaf name longer if it's already lengthy to avoid
+ // trouble with file name length limits.
+ suffix := "." + random.String(8)
+ var remoteSaved string
+ if len(path.Base(remote)) > 100 {
+ remoteSaved = TruncateString(remote, len(remote)-len(suffix)) + suffix
+ } else {
+ remoteSaved = remote + suffix
+ }
+
+ fs.Debugf(existingObj, "%s: renaming existing object to %q before starting", operation, remoteSaved)
+ existingObj, err = doMove(ctx, existingObj, remoteSaved)
+ if err != nil {
+ return nil, fmt.Errorf("%s: failed to rename existing file: %w", operation, err)
+ }
+ return func(perr *error) {
+ if *perr == nil {
+ fs.Debugf(existingObj, "%s: removing renamed existing file after operation", operation)
+ err := existingObj.Remove(ctx)
+ if err != nil {
+ *perr = fmt.Errorf("%s: failed to remove renamed existing file: %w", operation, err)
+ }
+ } else {
+ fs.Debugf(existingObj, "%s: renaming existing back after failed operation", operation)
+ _, renameErr := doMove(ctx, existingObj, remote)
+ if renameErr != nil {
+ fs.Errorf(existingObj, "%s: failed to restore existing file after failed operation: %v", operation, renameErr)
+ }
+ }
+ }, nil
+}
+
+// listToChan will transfer all objects in the listing to the output
+//
+// If an error occurs, the error will be logged, and it will close the
+// channel.
+//
+// If the error was ErrorDirNotFound then it will be ignored
+func listToChan(ctx context.Context, f fs.Fs, dir string) fs.ObjectsChan {
+ ci := fs.GetConfig(ctx)
+ o := make(fs.ObjectsChan, ci.Checkers)
+ go func() {
+ defer close(o)
+ err := walk.ListR(ctx, f, dir, true, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
+ entries.ForObject(func(obj fs.Object) {
+ o <- obj
+ })
+ return nil
+ })
+ if err != nil && err != fs.ErrorDirNotFound {
+ err = fmt.Errorf("failed to list: %w", err)
+ err = fs.CountError(ctx, err)
+ fs.Errorf(nil, "%v", err)
+ }
+ }()
+ return o
+}
+
+// CleanUp removes the trash for the Fs
+func CleanUp(ctx context.Context, f fs.Fs) error {
+ doCleanUp := f.Features().CleanUp
+ if doCleanUp == nil {
+ return fmt.Errorf("%v doesn't support cleanup", f)
+ }
+ if SkipDestructive(ctx, f, "clean up old files") {
+ return nil
+ }
+ return doCleanUp(ctx)
+}
+
+// wrap a Reader and a Closer together into a ReadCloser
+type readCloser struct {
+ io.Reader
+ io.Closer
+}
+
+// Cat any files to the io.Writer
+//
+// if offset == 0 it will be ignored
+// if offset > 0 then the file will be seeked to that offset
+// if offset < 0 then the file will be seeked that far from the end
+//
+// if count < 0 then it will be ignored
+// if count >= 0 then only that many characters will be output
+func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []byte) error {
+ var mu sync.Mutex
+ ci := fs.GetConfig(ctx)
+ return ListFn(ctx, f, func(o fs.Object) {
+ var err error
+ tr := accounting.Stats(ctx).NewTransfer(o, nil)
+ defer func() {
+ tr.Done(ctx, err)
+ }()
+ opt := fs.RangeOption{Start: offset, End: -1}
+ size := o.Size()
+ if opt.Start < 0 {
+ opt.Start += size
+ }
+ if count >= 0 {
+ opt.End = opt.Start + count - 1
+ }
+ var options []fs.OpenOption
+ if opt.Start > 0 || opt.End >= 0 {
+ options = append(options, &opt)
+ }
+ for _, option := range ci.DownloadHeaders {
+ options = append(options, option)
+ }
+ var in io.ReadCloser
+ in, err = Open(ctx, o, options...)
+ if err != nil {
+ err = fs.CountError(ctx, err)
+ fs.Errorf(o, "Failed to open: %v", err)
+ return
+ }
+ if count >= 0 {
+ in = &readCloser{Reader: &io.LimitedReader{R: in, N: count}, Closer: in}
+ }
+ in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer
+ // take the lock just before we output stuff, so at the last possible moment
+ mu.Lock()
+ defer mu.Unlock()
+ _, err = io.Copy(w, in)
+ if err != nil {
+ err = fs.CountError(ctx, err)
+ fs.Errorf(o, "Failed to send to output: %v", err)
+ }
+ if len(sep) > 0 {
+ _, err = w.Write(sep)
+ if err != nil {
+ err = fs.CountError(ctx, err)
+ fs.Errorf(o, "Failed to send separator to output: %v", err)
+ }
+ }
+ })
+}
+
+// Rcat reads data from the Reader until EOF and uploads it to a file on remote
+//
+// in is closed at the end of the transfer
+func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time, meta fs.Metadata) (dst fs.Object, err error) {
+ return rcatSrc(ctx, fdst, dstFileName, in, modTime, meta, nil)
+}
+
+// rcatSrc reads data from the Reader until EOF and uploads it to a file on remote
+//
+// in is closed at the end of the transfer
+//
+// Pass in fsrc if known or nil if not
+func rcatSrc(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time, meta fs.Metadata, fsrc fs.Fs) (dst fs.Object, err error) {
+ if SkipDestructive(ctx, dstFileName, "upload from pipe") {
+ // prevents "broken pipe" errors
+ _, err = io.Copy(io.Discard, in)
+ return nil, err
+ }
+
+ ci := fs.GetConfig(ctx)
+ tr := accounting.Stats(ctx).NewTransferRemoteSize(dstFileName, -1, nil, fdst)
+ defer func() {
+ tr.Done(ctx, err)
+ }()
+ var streamIn io.Reader = tr.Account(ctx, in).WithBuffer()
+
+ readCounter := readers.NewCountingReader(streamIn)
+ var trackingIn io.Reader
+ var hasher *hash.MultiHasher
+ var options []fs.OpenOption
+ if !ci.IgnoreChecksum {
+ hashes := hash.NewHashSet(fdst.Hashes().GetOne()) // just pick one hash
+ hashOption := &fs.HashesOption{Hashes: hashes}
+ options = append(options, hashOption)
+ hasher, err = hash.NewMultiHasherTypes(hashes)
+ if err != nil {
+ return nil, err
+ }
+ trackingIn = io.TeeReader(readCounter, hasher)
+ } else {
+ trackingIn = readCounter
+ }
+ for _, option := range ci.UploadHeaders {
+ options = append(options, option)
+ }
+ if ci.MetadataSet != nil {
+ options = append(options, fs.MetadataOption(ci.MetadataSet))
+ }
+
+ // get the sums from the hasher if in use, or nil
+ getSums := func() (sums map[hash.Type]string) {
+ if hasher != nil {
+ sums = hasher.Sums()
+ }
+ return sums
+ }
+
+ // Read the start of the input and check if it is small enough for direct upload
+ buf := make([]byte, ci.StreamingUploadCutoff)
+ fileIsSmall := false
+ if n, err := io.ReadFull(trackingIn, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
+ fileIsSmall = true
+ buf = buf[:n]
+ }
+
+ // Read the data we have already read in buf and any further unread
+ streamIn = io.MultiReader(bytes.NewReader(buf), trackingIn)
+
+ doPutStream := fdst.Features().PutStream
+
+ // Upload the input
+ if fileIsSmall || doPutStream == nil {
+ var rs io.ReadSeeker
+ if fileIsSmall {
+ fs.Debugf(fdst, "File to upload is small (%d bytes), uploading instead of streaming", len(buf))
+ rs = bytes.NewReader(buf)
+ } else {
+ fs.Debugf(fdst, "Target remote doesn't support streaming uploads, creating temporary local FS to spool file")
+ spool, err := os.CreateTemp("", "rclone-spool")
+ if err != nil {
+ return nil, fmt.Errorf("failed to create temporary spool file: %v", err)
+ }
+ fileName := spool.Name()
+ defer func() {
+ err := spool.Close()
+ if err != nil {
+ fs.Errorf(fileName, "Failed to close temporary spool file: %v", err)
+ }
+ err = os.Remove(fileName)
+ if err != nil {
+ fs.Errorf(fileName, "Failed to delete temporary spool file: %v", err)
+ }
+ }()
+ _, err = io.Copy(spool, streamIn)
+ if err != nil {
+ return nil, fmt.Errorf("failed to copy to temporary spool file: %v", err)
+ }
+ rs = spool
+ }
+ // Upload with Put with retries - since we have downloaded the file we know the size, and the hashes
+ sums := getSums()
+ size := int64(readCounter.BytesRead())
+ objInfo := object.NewStaticObjectInfo(dstFileName, modTime, size, false, sums, fsrc).WithMetadata(meta)
+ err = Retry(ctx, objInfo, ci.LowLevelRetries, func() error {
+ _, err = rs.Seek(0, io.SeekStart)
+ if err != nil {
+ return fmt.Errorf("failed to rewind temporary spool file: %v", err)
+ }
+ dst, err = fdst.Put(ctx, rs, objInfo, options...)
+ return err
+ })
+ } else {
+ // Upload with PutStream with no retries
+ objInfo := object.NewStaticObjectInfo(dstFileName, modTime, -1, false, nil, fsrc).WithMetadata(meta)
+ dst, err = doPutStream(ctx, streamIn, objInfo, options...)
+ }
+ if err != nil {
+ return dst, err
+ }
+
+ // Check transfer
+ sums := getSums()
+ opt := defaultEqualOpt(ctx)
+ if sums != nil {
+ // force --checksum on if we have hashes
+ opt.checkSum = true
+ }
+ src := object.NewStaticObjectInfo(dstFileName, modTime, int64(readCounter.BytesRead()), false, sums, fdst).WithMetadata(meta)
+ if !equal(ctx, src, dst, opt) {
+ err = fmt.Errorf("corrupted on transfer")
+ err = fs.CountError(ctx, err)
+ fs.Errorf(dst, "%v", err)
+ return dst, err
+ }
+ return dst, nil
+}
+
+// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
+func PublicLink(ctx context.Context, f fs.Fs, remote string, expire fs.Duration, unlink bool) (string, error) {
+ doPublicLink := f.Features().PublicLink
+ if doPublicLink == nil {
+ return "", fmt.Errorf("%v doesn't support public links", f)
+ }
+ return doPublicLink(ctx, remote, expire, unlink)
+}
+
+// Rmdirs removes any empty directories (or directories only
+// containing empty directories) under f, including f.
+//
+// Rmdirs obeys the filters
+func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
+ ci := fs.GetConfig(ctx)
+ fi := filter.GetConfig(ctx)
+ dirEmpty := make(map[string]bool)
+ dirEmpty[dir] = !leaveRoot
+ err := walk.Walk(ctx, f, dir, false, ci.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error {
+ if err != nil {
+ err = fs.CountError(ctx, err)
+ fs.Errorf(f, "Failed to list %q: %v", dirPath, err)
+ return nil
+ }
+ for _, entry := range entries {
+ switch x := entry.(type) {
+ case fs.Directory:
+ // add a new directory as empty
+ dir := x.Remote()
+ _, found := dirEmpty[dir]
+ if !found {
+ dirEmpty[dir] = true
+ }
+ case fs.Object:
+ // mark the parents of the file as being non-empty
+ dir := x.Remote()
+ for dir != "" {
+ dir = path.Dir(dir)
+ if dir == "." || dir == "/" {
+ dir = ""
+ }
+ empty, found := dirEmpty[dir]
+ // End if we reach a directory which is non-empty
+ if found && !empty {
+ break
+ }
+ dirEmpty[dir] = false
+ }
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return fmt.Errorf("failed to rmdirs: %w", err)
+ }
+
+ // Group directories to delete by level
+ var toDelete [][]string
+ for dir, empty := range dirEmpty {
+ if empty {
+ // If a filter matches the directory then that
+ // directory is a candidate for deletion
+ if fi.IncludeRemote(dir + "/") {
+ level := strings.Count(dir, "/") + 1
+ // The root directory "" is at the top level
+ if dir == "" {
+ level = 0
+ }
+ if len(toDelete) < level+1 {
+ toDelete = append(toDelete, make([][]string, level+1-len(toDelete))...)
+ }
+ toDelete[level] = append(toDelete[level], dir)
+ }
+ }
+ }
+
+ errCount := errcount.New()
+ // Delete all directories at the same level in parallel
+ for level := len(toDelete) - 1; level >= 0; level-- {
+ dirs := toDelete[level]
+ if len(dirs) == 0 {
+ continue
+ }
+ fs.Debugf(nil, "removing %d level %d directories", len(dirs), level)
+ sort.Strings(dirs)
+ g, gCtx := errgroup.WithContext(ctx)
+ g.SetLimit(ci.Checkers)
+ for _, dir := range dirs {
+ // End early if error
+ if gCtx.Err() != nil {
+ break
+ }
+ dir := dir
+ g.Go(func() error {
+ err := TryRmdir(gCtx, f, dir)
+ if err != nil {
+ err = fs.CountError(ctx, err)
+ fs.Errorf(dir, "Failed to rmdir: %v", err)
+ errCount.Add(err)
+ }
+ return nil // don't return errors, just count them
+ })
+ }
+ err := g.Wait()
+ if err != nil {
+ return err
+ }
+ }
+ return errCount.Err("failed to remove directories")
+}
+
+// GetCompareDest sets up --compare-dest
+func GetCompareDest(ctx context.Context) (CompareDest []fs.Fs, err error) {
+ ci := fs.GetConfig(ctx)
+ CompareDest, err = cache.GetArr(ctx, ci.CompareDest)
+ if err != nil {
+ return nil, fserrors.FatalError(fmt.Errorf("failed to make fs for --compare-dest %q: %w", ci.CompareDest, err))
+ }
+ return CompareDest, nil
+}
+
+// compareDest checks --compare-dest to see if src needs to
+// be copied
+//
+// Returns True if src is in --compare-dest
+func compareDest(ctx context.Context, dst, src fs.Object, CompareDest fs.Fs) (NoNeedTransfer bool, err error) {
+ var remote string
+ if dst == nil {
+ remote = src.Remote()
+ } else {
+ remote = dst.Remote()
+ }
+ CompareDestFile, err := CompareDest.NewObject(ctx, remote)
+ switch err {
+ case fs.ErrorObjectNotFound:
+ return false, nil
+ case nil:
+ break
+ default:
+ return false, err
+ }
+ opt := defaultEqualOpt(ctx)
+ opt.updateModTime = false
+ if equal(ctx, src, CompareDestFile, opt) {
+ fs.Debugf(src, "Destination found in --compare-dest, skipping")
+ return true, nil
+ }
+ return false, nil
+}
+
+// GetCopyDest sets up --copy-dest
+func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest []fs.Fs, err error) {
+ ci := fs.GetConfig(ctx)
+ CopyDest, err = cache.GetArr(ctx, ci.CopyDest)
+ if err != nil {
+ return nil, fserrors.FatalError(fmt.Errorf("failed to make fs for --copy-dest %q: %w", ci.CopyDest, err))
+ }
+ if !SameConfigArr(fdst, CopyDest) {
+ return nil, fserrors.FatalError(errors.New("parameter to --copy-dest has to be on the same remote as destination"))
+ }
+ for _, cf := range CopyDest {
+ if cf.Features().Copy == nil {
+ return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server side copy"))
+ }
+ }
+
+ return CopyDest, nil
+}
+
+// copyDest checks --copy-dest to see if src needs to
+// be copied
+//
+// Returns True if src was copied from --copy-dest
+func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
+ var remote string
+ if dst == nil {
+ remote = src.Remote()
+ } else {
+ remote = dst.Remote()
+ }
+ CopyDestFile, err := CopyDest.NewObject(ctx, remote)
+ switch err {
+ case fs.ErrorObjectNotFound:
+ return false, nil
+ case nil:
+ break
+ default:
+ return false, err
+ }
+ opt := defaultEqualOpt(ctx)
+ opt.updateModTime = false
+ if equal(ctx, src, CopyDestFile, opt) {
+ if dst == nil || !Equal(ctx, src, dst) {
+ if dst != nil && backupDir != nil {
+ err = MoveBackupDir(ctx, backupDir, dst)
+ if err != nil {
+ return false, fmt.Errorf("moving to --backup-dir failed: %w", err)
+ }
+ // If successful zero out the dstObj as it is no longer there
+ dst = nil
+ }
+ _, err := Copy(ctx, fdst, dst, remote, CopyDestFile)
+ if err != nil {
+ fs.Errorf(src, "Destination found in --copy-dest, error copying")
+ return false, nil
+ }
+ fs.Debugf(src, "Destination found in --copy-dest, using server-side copy")
+ return true, nil
+ }
+ fs.Debugf(src, "Unchanged skipping")
+ return true, nil
+ }
+ fs.Debugf(src, "Destination not found in --copy-dest")
+ return false, nil
+}
+
+// CompareOrCopyDest checks --compare-dest and --copy-dest to see if src
+// does not need to be copied
+//
+// Returns True if src does not need to be copied
+func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest []fs.Fs, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
+ ci := fs.GetConfig(ctx)
+ if len(ci.CompareDest) > 0 {
+ for _, compareF := range CompareOrCopyDest {
+ NoNeedTransfer, err := compareDest(ctx, dst, src, compareF)
+ if NoNeedTransfer || err != nil {
+ return NoNeedTransfer, err
+ }
+ }
+ } else if len(ci.CopyDest) > 0 {
+ for _, copyF := range CompareOrCopyDest {
+ NoNeedTransfer, err := copyDest(ctx, fdst, dst, src, copyF, backupDir)
+ if NoNeedTransfer || err != nil {
+ return NoNeedTransfer, err
+ }
+ }
+ }
+ return false, nil
+}
+
+// NeedTransfer checks to see if src needs to be copied to dst using
+// the current config.
+//
+// Returns a flag which indicates whether the file needs to be
+// transferred or not.
+func NeedTransfer(ctx context.Context, dst, src fs.Object) bool {
+ ci := fs.GetConfig(ctx)
+ logger, _ := GetLogger(ctx)
+ if dst == nil {
+ fs.Debugf(src, "Need to transfer - File not found at Destination")
+ logger(ctx, MissingOnDst, src, nil, nil)
+ return true
+ }
+ // If we should ignore existing files, don't transfer
+ if ci.IgnoreExisting {
+ fs.Debugf(src, "Destination exists, skipping")
+ logger(ctx, Match, src, dst, nil)
+ return false
+ }
+ // If we should upload unconditionally
+ if ci.IgnoreTimes {
+ fs.Debugf(src, "Transferring unconditionally as --ignore-times is in use")
+ logger(ctx, Differ, src, dst, nil)
+ return true
+ }
+ // If UpdateOlder is in effect, skip if dst is newer than src
+ if ci.UpdateOlder {
+ srcModTime := src.ModTime(ctx)
+ dstModTime := dst.ModTime(ctx)
+ dt := dstModTime.Sub(srcModTime)
+ // If have a mutually agreed precision then use that
+ modifyWindow := fs.GetModifyWindow(ctx, dst.Fs(), src.Fs())
+ if modifyWindow == fs.ModTimeNotSupported {
+ // Otherwise use 1 second as a safe default as
+ // the resolution of the time a file was
+ // uploaded.
+ modifyWindow = time.Second
+ }
+ switch {
+ case dt >= modifyWindow:
+ fs.Debugf(src, "Destination is newer than source, skipping")
+ logger(ctx, Match, src, dst, nil)
+ return false
+ case dt <= -modifyWindow:
+ // force --checksum on for the check and do update modtimes by default
+ opt := defaultEqualOpt(ctx)
+ opt.forceModTimeMatch = true
+ if equal(ctx, src, dst, opt) {
+ fs.Debugf(src, "Unchanged skipping")
+ return false
+ }
+ default:
+ // Do a size only compare unless --checksum is set
+ opt := defaultEqualOpt(ctx)
+ opt.sizeOnly = !ci.CheckSum
+ if equal(ctx, src, dst, opt) {
+ fs.Debugf(src, "Destination mod time is within %v of source and files identical, skipping", modifyWindow)
+ return false
+ }
+ fs.Debugf(src, "Destination mod time is within %v of source but files differ, transferring", modifyWindow)
+ }
+ } else {
+ // Check to see if changed or not
+ equalFn, ok := ctx.Value(equalFnKey).(EqualFn)
+ if ok {
+ return !equalFn(ctx, src, dst)
+ }
+ if Equal(ctx, src, dst) && !SameObject(src, dst) {
+ fs.Debugf(src, "Unchanged skipping")
+ return false
+ }
+ }
+ return true
+}
+
+// RcatSize reads data from the Reader until EOF and uploads it to a file on remote.
+// Pass in size >=0 if known, <0 if not known
+func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modTime time.Time, meta fs.Metadata) (dst fs.Object, err error) {
+ var obj fs.Object
+
+ if size >= 0 {
+ var err error
+ // Size known use Put
+ tr := accounting.Stats(ctx).NewTransferRemoteSize(dstFileName, size, nil, fdst)
+ defer func() {
+ tr.Done(ctx, err)
+ }()
+ body := io.NopCloser(in) // we let the server close the body
+ in := tr.Account(ctx, body) // account the transfer (no buffering)
+
+ if SkipDestructive(ctx, dstFileName, "upload from pipe") {
+ // prevents "broken pipe" errors
+ _, err = io.Copy(io.Discard, in)
+ return nil, err
+ }
+
+ info := object.NewStaticObjectInfo(dstFileName, modTime, size, true, nil, fdst).WithMetadata(meta)
+ obj, err = fdst.Put(ctx, in, info)
+ if err != nil {
+ fs.Errorf(dstFileName, "Post request put error: %v", err)
+
+ return nil, err
+ }
+ } else {
+ // Size unknown use Rcat
+ obj, err = Rcat(ctx, fdst, dstFileName, in, modTime, meta)
+ if err != nil {
+ fs.Errorf(dstFileName, "Post request rcat error: %v", err)
+
+ return nil, err
+ }
+ }
+
+ return obj, nil
+}
+
+// copyURLFunc is called from CopyURLFn
+type copyURLFunc func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error)
+
+// copyURLFn copies the data from the url to the function supplied
+func copyURLFn(ctx context.Context, dstFileName string, url string, autoFilename, dstFileNameFromHeader bool, fn copyURLFunc) (err error) {
+ client := fshttp.NewClient(ctx)
+ resp, err := client.Get(url)
+ if err != nil {
+ return err
+ }
+ defer fs.CheckClose(resp.Body, &err)
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return fmt.Errorf("CopyURL failed: %s", resp.Status)
+ }
+ modTime, err := http.ParseTime(resp.Header.Get("Last-Modified"))
+ if err != nil {
+ modTime = time.Now()
+ }
+ if autoFilename {
+ if dstFileNameFromHeader {
+ _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition"))
+ headerFilename := path.Base(strings.ReplaceAll(params["filename"], "\\", "/"))
+ if err != nil || headerFilename == "" {
+ return fmt.Errorf("CopyURL failed: filename not found in the Content-Disposition header")
+ }
+ fs.Debugf(headerFilename, "filename found in Content-Disposition header.")
+ return fn(ctx, headerFilename, resp.Body, resp.ContentLength, modTime)
+ }
+
+ dstFileName = path.Base(resp.Request.URL.Path)
+ if dstFileName == "." || dstFileName == "/" {
+ return fmt.Errorf("CopyURL failed: file name wasn't found in url")
+ }
+ fs.Debugf(dstFileName, "File name found in url")
+ }
+ return fn(ctx, dstFileName, resp.Body, resp.ContentLength, modTime)
+}
+
+// CopyURL copies the data from the url to (fdst, dstFileName)
+func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, autoFilename, dstFileNameFromHeader bool, noClobber bool) (dst fs.Object, err error) {
+ err = copyURLFn(ctx, dstFileName, url, autoFilename, dstFileNameFromHeader, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) {
+ if noClobber {
+ _, err = fdst.NewObject(ctx, dstFileName)
+ if err == nil {
+ return errors.New("CopyURL failed: file already exist")
+ }
+ }
+ dst, err = RcatSize(ctx, fdst, dstFileName, in, size, modTime, nil)
+ return err
+ })
+ return dst, err
+}
+
+// CopyURLToWriter copies the data from the url to the io.Writer supplied
+func CopyURLToWriter(ctx context.Context, url string, out io.Writer) (err error) {
+ return copyURLFn(ctx, "", url, false, false, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) {
+ _, err = io.Copy(out, in)
+ return err
+ })
+}
+
+// BackupDir returns the correctly configured --backup-dir
+func BackupDir(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, srcFileName string) (backupDir fs.Fs, err error) {
+ ci := fs.GetConfig(ctx)
+ if ci.BackupDir != "" {
+ backupDir, err = cache.Get(ctx, ci.BackupDir)
+ if err != nil {
+ return nil, fserrors.FatalError(fmt.Errorf("failed to make fs for --backup-dir %q: %w", ci.BackupDir, err))
+ }
+ if !SameConfig(fdst, backupDir) {
+ return nil, fserrors.FatalError(errors.New("parameter to --backup-dir has to be on the same remote as destination"))
+ }
+ if srcFileName == "" {
+ if OverlappingFilterCheck(ctx, backupDir, fdst) {
+ return nil, fserrors.FatalError(errors.New("destination and parameter to --backup-dir mustn't overlap"))
+ }
+ if OverlappingFilterCheck(ctx, backupDir, fsrc) {
+ return nil, fserrors.FatalError(errors.New("source and parameter to --backup-dir mustn't overlap"))
+ }
+ } else if ci.Suffix == "" {
+ if SameDir(fdst, backupDir) {
+ return nil, fserrors.FatalError(errors.New("destination and parameter to --backup-dir mustn't be the same"))
+ }
+ if SameDir(fsrc, backupDir) {
+ return nil, fserrors.FatalError(errors.New("source and parameter to --backup-dir mustn't be the same"))
+ }
+ }
+ } else if ci.Suffix != "" {
+ // --backup-dir is not set but --suffix is - use the destination as the backupDir
+ backupDir = fdst
+ } else {
+ return nil, fserrors.FatalError(errors.New("internal error: BackupDir called when --backup-dir and --suffix both empty"))
+ }
+ if !CanServerSideMove(backupDir) {
+ return nil, fserrors.FatalError(errors.New("can't use --backup-dir on a remote which doesn't support server-side move or copy"))
+ }
+ return backupDir, nil
+}
+
+// MoveBackupDir moves a file to the backup dir
+func MoveBackupDir(ctx context.Context, backupDir fs.Fs, dst fs.Object) (err error) {
+ remoteWithSuffix := SuffixName(ctx, dst.Remote())
+ overwritten, _ := backupDir.NewObject(ctx, remoteWithSuffix)
+ _, err = Move(ctx, backupDir, overwritten, remoteWithSuffix, dst)
+ return err
+}
+
+// needsMoveCaseInsensitive returns true if moveCaseInsensitive is needed
+func needsMoveCaseInsensitive(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) bool {
+ dstFilePath := path.Join(fdst.Root(), dstFileName)
+ srcFilePath := path.Join(fsrc.Root(), srcFileName)
+ if !cp && fdst.Name() == fsrc.Name() && dstFileName != srcFileName && norm.NFC.String(dstFilePath) == norm.NFC.String(srcFilePath) {
+ return true
+ }
+ return !cp && fdst.Name() == fsrc.Name() && fdst.Features().CaseInsensitive && dstFileName != srcFileName && strings.EqualFold(dstFilePath, srcFilePath)
+}
+
+// MoveCaseInsensitive handles changing case of a file on a case insensitive remote.
+// This will move the file to a temporary name then
+// move it back to the intended destination. This is required
+// to avoid issues with certain remotes and avoid file deletion.
+// returns nil, nil if !needsMoveCaseInsensitive.
+// this does not account a transfer -- the caller should do that if desired.
+func MoveCaseInsensitive(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool, srcObj fs.Object) (newDst fs.Object, err error) {
+ logger, _ := GetLogger(ctx)
+
+ // Choose operations
+ Op := MoveTransfer
+ if cp {
+ Op = Copy
+ }
+
+ if SkipDestructive(ctx, srcFileName, "rename to "+dstFileName) {
+ // avoid fatalpanic on --dry-run (trying to access non-existent tmpObj)
+ return nil, nil
+ }
+ // Create random name to temporarily move file to
+ tmpObjName := dstFileName + "-rclone-move-" + random.String(8)
+ tmpObjFail, err := fdst.NewObject(ctx, tmpObjName)
+ if err != fs.ErrorObjectNotFound {
+ if err == nil {
+ logger(ctx, TransferError, nil, tmpObjFail, err)
+ return nil, errors.New("found an already existing file with a randomly generated name. Try the operation again")
+ }
+ logger(ctx, TransferError, nil, tmpObjFail, err)
+ return nil, fmt.Errorf("error while attempting to move file to a temporary location: %w", err)
+ }
+ fs.Debugf(srcObj, "moving to %v", tmpObjName)
+ tmpObj, err := Op(ctx, fdst, nil, tmpObjName, srcObj)
+ if err != nil {
+ logger(ctx, TransferError, srcObj, tmpObj, err)
+ return nil, fmt.Errorf("error while moving file to temporary location: %w", err)
+ }
+ fs.Debugf(srcObj, "moving to %v", dstFileName)
+ newDst, err = Op(ctx, fdst, nil, dstFileName, tmpObj)
+ logger(ctx, MissingOnDst, tmpObj, nil, err)
+ return newDst, err
+}
+
+// moveOrCopyFile moves or copies a single file possibly to a new name
+func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool, allowOverlap bool) (err error) {
+ ci := fs.GetConfig(ctx)
+ logger, usingLogger := GetLogger(ctx)
+ dstFilePath := path.Join(fdst.Root(), dstFileName)
+ srcFilePath := path.Join(fsrc.Root(), srcFileName)
+ if fdst.Name() == fsrc.Name() && dstFilePath == srcFilePath && !allowOverlap {
+ fs.Debugf(fdst, "don't need to copy/move %s, it is already at target location", dstFileName)
+ if usingLogger {
+ srcObj, _ := fsrc.NewObject(ctx, srcFileName)
+ dstObj, _ := fsrc.NewObject(ctx, dstFileName)
+ logger(ctx, Match, srcObj, dstObj, nil)
+ }
+ return nil
+ }
+
+ // Choose operations
+ Op := MoveTransfer
+ if cp {
+ Op = Copy
+ }
+
+ // Find src object
+ srcObj, err := fsrc.NewObject(ctx, srcFileName)
+ if err != nil {
+ logger(ctx, TransferError, srcObj, nil, err)
+ return err
+ }
+
+ // Find dst object if it exists
+ var dstObj fs.Object
+ if !ci.NoCheckDest {
+ dstObj, err = fdst.NewObject(ctx, dstFileName)
+ if errors.Is(err, fs.ErrorObjectNotFound) {
+ dstObj = nil
+ } else if err != nil {
+ logger(ctx, TransferError, nil, dstObj, err)
+ return err
+ }
+ }
+
+ // Special case for changing case of a file on a case insensitive remote
+ // This will move the file to a temporary name then
+ // move it back to the intended destination. This is required
+ // to avoid issues with certain remotes and avoid file deletion.
+ if needsMoveCaseInsensitive(fdst, fsrc, dstFileName, srcFileName, cp) {
+ tr := accounting.Stats(ctx).NewTransfer(srcObj, fdst)
+ defer func() {
+ tr.Done(ctx, err)
+ }()
+ _, err = MoveCaseInsensitive(ctx, fdst, fsrc, dstFileName, srcFileName, cp, srcObj)
+ return err
+ }
+
+ var backupDir fs.Fs
+ var copyDestDir []fs.Fs
+ if ci.BackupDir != "" || ci.Suffix != "" {
+ backupDir, err = BackupDir(ctx, fdst, fsrc, srcFileName)
+ if err != nil {
+ return fmt.Errorf("creating Fs for --backup-dir failed: %w", err)
+ }
+ }
+ if len(ci.CompareDest) > 0 {
+ copyDestDir, err = GetCompareDest(ctx)
+ if err != nil {
+ return err
+ }
+ } else if len(ci.CopyDest) > 0 {
+ copyDestDir, err = GetCopyDest(ctx, fdst)
+ if err != nil {
+ return err
+ }
+ }
+ needTransfer := NeedTransfer(ctx, dstObj, srcObj)
+ if needTransfer {
+ NoNeedTransfer, err := CompareOrCopyDest(ctx, fdst, dstObj, srcObj, copyDestDir, backupDir)
+ if err != nil {
+ return err
+ }
+ if NoNeedTransfer {
+ needTransfer = false
+ }
+ }
+ if needTransfer {
+ // If destination already exists, then we must move it into --backup-dir if required
+ if dstObj != nil && backupDir != nil {
+ err = MoveBackupDir(ctx, backupDir, dstObj)
+ if err != nil {
+ logger(ctx, TransferError, dstObj, nil, err)
+ return fmt.Errorf("moving to --backup-dir failed: %w", err)
+ }
+ // If successful zero out the dstObj as it is no longer there
+ logger(ctx, MissingOnDst, dstObj, nil, nil)
+ dstObj = nil
+ }
+
+ _, err = Op(ctx, fdst, dstObj, dstFileName, srcObj)
+ } else if !cp {
+ if ci.IgnoreExisting {
+ fs.Debugf(srcObj, "Not removing source file as destination file exists and --ignore-existing is set")
+ logger(ctx, Match, srcObj, dstObj, nil)
+ } else if !SameObject(srcObj, dstObj) {
+ err = DeleteFile(ctx, srcObj)
+ logger(ctx, Differ, srcObj, dstObj, nil)
+ }
+ }
+ return err
+}
+
+// MoveFile moves a single file possibly to a new name
+//
+// This is treated as a transfer.
+func MoveFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) {
+ return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, false, false)
+}
+
+// TransformFile transforms a file in place using --name-transform
+//
+// This is treated as a transfer.
+func TransformFile(ctx context.Context, fdst fs.Fs, srcFileName string) (err error) {
+ return moveOrCopyFile(ctx, fdst, fdst, srcFileName, srcFileName, false, true)
+}
+
+// SetTier changes tier of object in remote
+func SetTier(ctx context.Context, fsrc fs.Fs, tier string) error {
+ return ListFn(ctx, fsrc, func(o fs.Object) {
+ objImpl, ok := o.(fs.SetTierer)
+ if !ok {
+ fs.Errorf(fsrc, "Remote object does not implement SetTier")
+ return
+ }
+ err := objImpl.SetTier(tier)
+ if err != nil {
+ fs.Errorf(fsrc, "Failed to do SetTier, %v", err)
+ }
+ })
+}
+
+// SetTierFile changes tier of a single file in remote
+func SetTierFile(ctx context.Context, o fs.Object, tier string) error {
+ do, ok := o.(fs.SetTierer)
+ if !ok {
+ return errors.New("remote object does not implement SetTier")
+ }
+ err := do.SetTier(tier)
+ if err != nil {
+ fs.Errorf(o, "Failed to do SetTier, %v", err)
+ return err
+ }
+ return nil
+}
+
+// TouchDir touches every file in directory with time t
+func TouchDir(ctx context.Context, f fs.Fs, remote string, t time.Time, recursive bool) error {
+ ci := fs.GetConfig(ctx)
+ g, gCtx := errgroup.WithContext(ctx)
+ g.SetLimit(ci.Transfers)
+ err := walk.ListR(ctx, f, remote, false, ConfigMaxDepth(ctx, recursive), walk.ListObjects, func(entries fs.DirEntries) error {
+ entries.ForObject(func(o fs.Object) {
+ if !SkipDestructive(ctx, o, "touch") {
+ g.Go(func() error {
+ fs.Debugf(f, "Touching %q", o.Remote())
+ err := o.SetModTime(gCtx, t)
+ if err != nil {
+ err = fmt.Errorf("failed to touch: %w", err)
+ err = fs.CountError(gCtx, err)
+ fs.Errorf(o, "%v", err)
+ }
+ return nil
+ })
+ }
+ })
+ return nil
+ })
+ _ = g.Wait()
+ return err
+}
+
+// ListFormat defines files information print format
+type ListFormat struct {
+ separator string
+ dirSlash bool
+ absolute bool
+ output []func(entry *ListJSONItem) string
+ csv *csv.Writer
+ buf bytes.Buffer
+}
+
+// SetSeparator changes separator in struct
+func (l *ListFormat) SetSeparator(separator string) {
+ l.separator = separator
+}
+
+// SetDirSlash defines if slash should be printed
+func (l *ListFormat) SetDirSlash(dirSlash bool) {
+ l.dirSlash = dirSlash
+}
+
+// SetAbsolute prints a leading slash in front of path names
+func (l *ListFormat) SetAbsolute(absolute bool) {
+ l.absolute = absolute
+}
+
+// SetCSV defines if the output should be csv
+//
+// Note that you should call SetSeparator before this if you want a
+// custom separator
+func (l *ListFormat) SetCSV(useCSV bool) {
+ if useCSV {
+ l.csv = csv.NewWriter(&l.buf)
+ if l.separator != "" {
+ l.csv.Comma = []rune(l.separator)[0]
+ }
+ } else {
+ l.csv = nil
+ }
+}
+
+// SetOutput sets functions used to create files information
+func (l *ListFormat) SetOutput(output []func(entry *ListJSONItem) string) {
+ l.output = output
+}
+
+// AddModTime adds file's Mod Time to output
+func (l *ListFormat) AddModTime(timeFormat string) {
+ switch timeFormat {
+ case "":
+ l.AppendOutput(func(entry *ListJSONItem) string {
+ return entry.ModTime.When.Local().Format("2006-01-02 15:04:05")
+ })
+ case "unix":
+ l.AppendOutput(func(entry *ListJSONItem) string {
+ return fmt.Sprint(entry.ModTime.When.Unix())
+ })
+ case "unixnano":
+ l.AppendOutput(func(entry *ListJSONItem) string {
+ return fmt.Sprint(entry.ModTime.When.UnixNano())
+ })
+ default:
+ timeFormat = transform.TimeFormat(timeFormat)
+ l.AppendOutput(func(entry *ListJSONItem) string {
+ return entry.ModTime.When.Local().Format(timeFormat)
+ })
+ }
+}
+
+// AddSize adds file's size to output
+func (l *ListFormat) AddSize() {
+ l.AppendOutput(func(entry *ListJSONItem) string {
+ return strconv.FormatInt(entry.Size, 10)
+ })
+}
+
+// normalisePath makes sure the path has the correct slashes for the current mode
+func (l *ListFormat) normalisePath(entry *ListJSONItem, remote string) string {
+ if l.absolute && !strings.HasPrefix(remote, "/") {
+ remote = "/" + remote
+ }
+ if entry.IsDir && l.dirSlash {
+ remote += "/"
+ }
+ return remote
+}
+
+// AddPath adds path to file to output
+func (l *ListFormat) AddPath() {
+ l.AppendOutput(func(entry *ListJSONItem) string {
+ return l.normalisePath(entry, entry.Path)
+ })
+}
+
+// AddEncrypted adds the encrypted path to file to output
+func (l *ListFormat) AddEncrypted() {
+ l.AppendOutput(func(entry *ListJSONItem) string {
+ return l.normalisePath(entry, entry.Encrypted)
+ })
+}
+
+// AddHash adds the hash of the type given to the output
+func (l *ListFormat) AddHash(ht hash.Type) {
+ hashName := ht.String()
+ l.AppendOutput(func(entry *ListJSONItem) string {
+ if entry.IsDir {
+ return ""
+ }
+ return entry.Hashes[hashName]
+ })
+}
+
+// AddID adds file's ID to the output if known
+func (l *ListFormat) AddID() {
+ l.AppendOutput(func(entry *ListJSONItem) string {
+ return entry.ID
+ })
+}
+
+// AddOrigID adds file's Original ID to the output if known
+func (l *ListFormat) AddOrigID() {
+ l.AppendOutput(func(entry *ListJSONItem) string {
+ return entry.OrigID
+ })
+}
+
+// AddTier adds file's Tier to the output if known
+func (l *ListFormat) AddTier() {
+ l.AppendOutput(func(entry *ListJSONItem) string {
+ return entry.Tier
+ })
+}
+
+// AddMimeType adds file's MimeType to the output if known
+func (l *ListFormat) AddMimeType() {
+ l.AppendOutput(func(entry *ListJSONItem) string {
+ return entry.MimeType
+ })
+}
+
+// AddMetadata adds file's Metadata to the output if known
+func (l *ListFormat) AddMetadata() {
+ l.AppendOutput(func(entry *ListJSONItem) string {
+ metadata := entry.Metadata
+ if metadata == nil {
+ metadata = make(fs.Metadata)
+ }
+ out, err := json.Marshal(metadata)
+ if err != nil {
+ return fmt.Sprintf("Failed to read metadata: %v", err.Error())
+ }
+ return string(out)
+ })
+}
+
+// AppendOutput adds string generated by specific function to printed output
+func (l *ListFormat) AppendOutput(functionToAppend func(item *ListJSONItem) string) {
+ l.output = append(l.output, functionToAppend)
+}
+
+// Format prints information about the DirEntry in the format defined
+func (l *ListFormat) Format(entry *ListJSONItem) (result string) {
+ var out []string
+ for _, fun := range l.output {
+ out = append(out, fun(entry))
+ }
+ if l.csv != nil {
+ l.buf.Reset()
+ _ = l.csv.Write(out) // can't fail writing to bytes.Buffer
+ l.csv.Flush()
+ result = strings.TrimRight(l.buf.String(), "\n")
+ } else {
+ result = strings.Join(out, l.separator)
+ }
+ return result
+}
+
+// FormatForLSFPrecision Returns a time format for the given precision
+func FormatForLSFPrecision(precision time.Duration) string {
+ switch {
+ case precision <= time.Nanosecond:
+ return "2006-01-02 15:04:05.000000000"
+ case precision <= 10*time.Nanosecond:
+ return "2006-01-02 15:04:05.00000000"
+ case precision <= 100*time.Nanosecond:
+ return "2006-01-02 15:04:05.0000000"
+ case precision <= time.Microsecond:
+ return "2006-01-02 15:04:05.000000"
+ case precision <= 10*time.Microsecond:
+ return "2006-01-02 15:04:05.00000"
+ case precision <= 100*time.Microsecond:
+ return "2006-01-02 15:04:05.0000"
+ case precision <= time.Millisecond:
+ return "2006-01-02 15:04:05.000"
+ case precision <= 10*time.Millisecond:
+ return "2006-01-02 15:04:05.00"
+ case precision <= 100*time.Millisecond:
+ return "2006-01-02 15:04:05.0"
+ }
+ return "2006-01-02 15:04:05"
+}
+
+// DirMove renames srcRemote to dstRemote
+//
+// It does this by loading the directory tree into memory (using ListR
+// if available) and doing renames in parallel.
+func DirMove(ctx context.Context, f fs.Fs, srcRemote, dstRemote string) (err error) {
+ ci := fs.GetConfig(ctx)
+
+ if SkipDestructive(ctx, srcRemote, "dirMove") {
+ accounting.Stats(ctx).Renames(1)
+ return nil
+ }
+
+ // Use DirMove if possible
+ if doDirMove := f.Features().DirMove; doDirMove != nil {
+ err = doDirMove(ctx, f, srcRemote, dstRemote)
+ if err == nil {
+ accounting.Stats(ctx).Renames(1)
+ }
+ if err != fs.ErrorCantDirMove && err != fs.ErrorDirExists {
+ return err
+ }
+ fs.Infof(f, "Can't DirMove - falling back to file moves: %v", err)
+ }
+
+ // Load the directory tree into memory
+ tree, err := walk.NewDirTree(ctx, f, srcRemote, true, -1)
+ if err != nil {
+ return fmt.Errorf("RenameDir tree walk: %w", err)
+ }
+
+ // Get the directories in sorted order
+ dirs := tree.Dirs()
+
+ // Make the destination directories - must be done in order not in parallel
+ for _, dir := range dirs {
+ dstPath := dstRemote + dir[len(srcRemote):]
+ err := f.Mkdir(ctx, dstPath)
+ if err != nil {
+ return fmt.Errorf("RenameDir mkdir: %w", err)
+ }
+ }
+
+ // Rename the files in parallel
+ type rename struct {
+ o fs.Object
+ newPath string
+ }
+ renames := make(chan rename, ci.Checkers)
+ g, gCtx := errgroup.WithContext(context.Background())
+ for range ci.Checkers {
+ g.Go(func() error {
+ for job := range renames {
+ dstOverwritten, _ := f.NewObject(gCtx, job.newPath)
+ _, err := Move(gCtx, f, dstOverwritten, job.newPath, job.o)
+ if err != nil {
+ return err
+ }
+ select {
+ case <-gCtx.Done():
+ return gCtx.Err()
+ default:
+ }
+
+ }
+ return nil
+ })
+ }
+ for dir, entries := range tree {
+ dstPath := dstRemote + dir[len(srcRemote):]
+ for _, entry := range entries {
+ if o, ok := entry.(fs.Object); ok {
+ renames <- rename{o, path.Join(dstPath, path.Base(o.Remote()))}
+ }
+ }
+ }
+ close(renames)
+ err = g.Wait()
+ if err != nil {
+ return fmt.Errorf("RenameDir renames: %w", err)
+ }
+
+ // Remove the source directories in reverse order
+ for i := len(dirs) - 1; i >= 0; i-- {
+ err := f.Rmdir(ctx, dirs[i])
+ if err != nil {
+ return fmt.Errorf("RenameDir rmdir: %w", err)
+ }
+ }
+
+ return nil
+}
+
+// DirMoveCaseInsensitive does DirMove in two steps (to temp name, then real name)
+// which is necessary for some case-insensitive backends
+func DirMoveCaseInsensitive(ctx context.Context, f fs.Fs, srcRemote, dstRemote string) (err error) {
+ tmpDstRemote := dstRemote + "-rclone-move-" + random.String(8)
+ err = DirMove(ctx, f, srcRemote, tmpDstRemote)
+ if err != nil {
+ return err
+ }
+ return DirMove(ctx, f, tmpDstRemote, dstRemote)
+}
+
+// FsInfo provides information about a remote
+type FsInfo struct {
+ // Name of the remote (as passed into NewFs)
+ Name string
+
+ // Root of the remote (as passed into NewFs)
+ Root string
+
+ // String returns a description of the FS
+ String string
+
+ // Precision of the ModTimes in this Fs in Nanoseconds
+ Precision time.Duration
+
+ // Returns the supported hash types of the filesystem
+ Hashes []string
+
+ // Features returns the optional features of this Fs
+ Features map[string]bool
+
+ // MetadataInfo returns info about the metadata for this backend
+ MetadataInfo *fs.MetadataInfo
+}
+
+// GetFsInfo gets the information (FsInfo) about a given Fs
+func GetFsInfo(f fs.Fs) *FsInfo {
+ features := f.Features()
+ info := &FsInfo{
+ Name: f.Name(),
+ Root: f.Root(),
+ String: f.String(),
+ Precision: f.Precision(),
+ Hashes: make([]string, 0, 4),
+ Features: features.Enabled(),
+ MetadataInfo: nil,
+ }
+ for _, hashType := range f.Hashes().Array() {
+ info.Hashes = append(info.Hashes, hashType.String())
+ }
+ fsInfo, _, _, _, err := fs.ParseRemote(fs.ConfigString(f))
+ if err == nil && fsInfo != nil && fsInfo.MetadataInfo != nil {
+ info.MetadataInfo = fsInfo.MetadataInfo
+ }
+ return info
+}
+
+var (
+ interactiveMu sync.Mutex // protects the following variables
+ skipped = map[string]bool{}
+)
+
+// skipDestructiveChoose asks the user which action to take
+//
+// Call with interactiveMu held
+func skipDestructiveChoose(ctx context.Context, subject any, action string) (skip bool) {
+ // Lock the StdoutMutex - must not call fs.Log anything
+ // otherwise it will deadlock with --interactive --progress
+ StdoutMutex.Lock()
+
+ fmt.Printf("\nrclone: %s \"%v\"?\n", action, subject)
+ i := config.CommandDefault([]string{
+ "yYes, this is OK",
+ "nNo, skip this",
+ fmt.Sprintf("sSkip all %s operations with no more questions", action),
+ fmt.Sprintf("!Do all %s operations with no more questions", action),
+ "qExit rclone now.",
+ }, 0)
+
+ StdoutMutex.Unlock()
+
+ switch i {
+ case 'y':
+ skip = false
+ case 'n':
+ skip = true
+ case 's':
+ skip = true
+ skipped[action] = true
+ fs.Logf(nil, "Skipping all %s operations from now on without asking", action)
+ case '!':
+ skip = false
+ skipped[action] = false
+ fs.Logf(nil, "Doing all %s operations from now on without asking", action)
+ case 'q':
+ fs.Logf(nil, "Quitting rclone now")
+ atexit.Run()
+ os.Exit(0)
+ default:
+ skip = true
+ fs.Errorf(nil, "Bad choice %c", i)
+ }
+ return skip
+}
+
+// SkipDestructive should be called whenever rclone is about to do an destructive operation.
+//
+// It will check the --dry-run flag and it will ask the user if the --interactive flag is set.
+//
+// subject should be the object or directory in use
+//
+// action should be a descriptive word or short phrase
+//
+// Together they should make sense in this sentence: "Rclone is about
+// to action subject".
+func SkipDestructive(ctx context.Context, subject any, action string) (skip bool) {
+ var flag string
+ ci := fs.GetConfig(ctx)
+ switch {
+ case ci.DryRun:
+ flag = "--dry-run"
+ skip = true
+ case ci.Interactive:
+ flag = "--interactive"
+ interactiveMu.Lock()
+ defer interactiveMu.Unlock()
+ var found bool
+ skip, found = skipped[action]
+ if !found {
+ skip = skipDestructiveChoose(ctx, subject, action)
+ }
+ default:
+ return false
+ }
+ if skip {
+ size := int64(-1)
+ if do, ok := subject.(interface{ Size() int64 }); ok {
+ size = do.Size()
+ }
+ if size >= 0 {
+ fs.Logf(subject, "Skipped %s as %s is set (size %v)", fs.LogValue("skipped", action), flag, fs.LogValue("size", fs.SizeSuffix(size)))
+ } else {
+ fs.Logf(subject, "Skipped %s as %s is set", fs.LogValue("skipped", action), flag)
+ }
+ }
+ return skip
+}
+
+// Return the best way of describing the directory for the logs
+func dirName(f fs.Fs, dst fs.Directory, dir string) any {
+ if dst != nil {
+ if dst.Remote() != "" {
+ return dst
+ }
+ // Root is described as the Fs
+ return f
+ }
+ if dir != "" {
+ return dir
+ }
+ // Root is described as the Fs
+ return f
+}
+
+// CopyDirMetadata copies the src directory to dst or f if nil. If dst is nil then it uses
+// dir as the name of the new directory.
+//
+// It returns the destination directory if possible. Note that this may
+// be nil.
+func CopyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory, err error) {
+ ci := fs.GetConfig(ctx)
+ logName := dirName(f, dst, dir)
+ if SkipDestructive(ctx, logName, "update directory metadata") {
+ return nil, nil
+ }
+
+ // Options for the directory metadata
+ options := []fs.OpenOption{}
+ if ci.MetadataSet != nil {
+ options = append(options, fs.MetadataOption(ci.MetadataSet))
+ }
+
+ // Read metadata from src and add options and use metadata mapper
+ metadata, err := fs.GetMetadataOptions(ctx, f, src, options)
+ if err != nil {
+ return nil, err
+ }
+
+ // Fall back to ModTime if metadata not available
+ if metadata == nil {
+ metadata = fs.Metadata{}
+ }
+ if metadata["mtime"] == "" {
+ metadata["mtime"] = src.ModTime(ctx).Format(time.RFC3339Nano)
+ }
+
+ // Now set the metadata
+ if dst == nil {
+ do := f.Features().MkdirMetadata
+ if do == nil {
+ return nil, fmt.Errorf("internal error: expecting %v to have MkdirMetadata method: %w", f, fs.ErrorNotImplemented)
+ }
+ newDst, err = do(ctx, dir, metadata)
+ } else {
+ do, ok := dst.(fs.SetMetadataer)
+ if !ok {
+ return nil, fmt.Errorf("internal error: expecting directory %s (%T) from %v to have SetMetadata method: %w", logName, dst, f, fs.ErrorNotImplemented)
+ }
+ err = do.SetMetadata(ctx, metadata)
+ newDst = dst
+ }
+ if err != nil {
+ return nil, err
+ }
+ fs.Infof(logName, "Updated directory metadata")
+ return newDst, nil
+}
+
+// SetDirModTime sets the modtime on dst or dir
+//
+// If dst is nil then it uses dir as the name of the directory.
+//
+// It returns the destination directory if possible. Note that this
+// may be nil.
+//
+// It does not create the directory.
+func SetDirModTime(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, modTime time.Time) (newDst fs.Directory, err error) {
+ logName := dirName(f, dst, dir)
+ ci := fs.GetConfig(ctx)
+ if ci.NoUpdateDirModTime {
+ fs.Debugf(logName, "Skipping set directory modification time as --no-update-dir-modtime is set")
+ return nil, nil
+ }
+ if SkipDestructive(ctx, logName, "set directory modification time") {
+ return nil, nil
+ }
+ if dst != nil {
+ dir = dst.Remote()
+ }
+
+ // Try to set the ModTime with the Directory.SetModTime method first as this is the most efficient
+ if dst != nil {
+ if do, ok := dst.(fs.SetModTimer); ok {
+ err := do.SetModTime(ctx, modTime)
+ if errors.Is(err, fs.ErrorNotImplemented) {
+ // Fall through and run the code below if not implemented
+ // This can happen for fs.DirWrapper instances
+ } else if err != nil {
+ return dst, err
+ } else {
+ fs.Infof(logName, "Set directory modification time (using SetModTime)")
+ return dst, nil
+ }
+ }
+ }
+
+ // Next try to set the ModTime with the Fs.DirSetModTime method as this works for non-metadata backends
+ if do := f.Features().DirSetModTime; do != nil {
+ err := do(ctx, dir, modTime)
+ if err != nil {
+ return dst, err
+ }
+ fs.Infof(logName, "Set directory modification time (using DirSetModTime)")
+ return dst, nil
+ }
+
+ // Something should have worked so return an error
+ return nil, fmt.Errorf("no method to set directory modtime found for %v (%T): %w", f, dst, fs.ErrorNotImplemented)
+}
diff --git a/fs/operations/operations_internal_test.go b/fs/operations/operations_internal_test.go
new file mode 100644
index 0000000..658249e
--- /dev/null
+++ b/fs/operations/operations_internal_test.go
@@ -0,0 +1,44 @@
+// Internal tests for operations
+
+package operations
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ _ "github.com/mewsen/rclone-studip-backend-oot/backend/studip"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/object"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestSizeDiffers(t *testing.T) {
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
+ when := time.Now()
+ for _, test := range []struct {
+ ignoreSize bool
+ srcSize int64
+ dstSize int64
+ want bool
+ }{
+ {false, 0, 0, false},
+ {false, 1, 2, true},
+ {false, 1, -1, false},
+ {false, -1, 1, false},
+ {true, 0, 0, false},
+ {true, 1, 2, false},
+ {true, 1, -1, false},
+ {true, -1, 1, false},
+ } {
+ src := object.NewStaticObjectInfo("a", when, test.srcSize, true, nil, nil)
+ dst := object.NewStaticObjectInfo("a", when, test.dstSize, true, nil, nil)
+ oldIgnoreSize := ci.IgnoreSize
+ ci.IgnoreSize = test.ignoreSize
+ got := sizeDiffers(ctx, src, dst)
+ ci.IgnoreSize = oldIgnoreSize
+ assert.Equal(t, test.want, got, fmt.Sprintf("ignoreSize=%v, srcSize=%v, dstSize=%v", test.ignoreSize, test.srcSize, test.dstSize))
+ }
+}
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
new file mode 100644
index 0000000..3fb5992
--- /dev/null
+++ b/fs/operations/operations_test.go
@@ -0,0 +1,1968 @@
+// Integration tests - test rclone by doing real transactions to a
+// storage provider to and from the local disk.
+//
+// By default it will use a local fs, however you can provide a
+// -remote option to use a different remote. The test_all.go script
+// is a wrapper to call this for all the test remotes.
+//
+// FIXME not safe for concurrent running of tests until fs.Config is
+// no longer a global
+//
+// NB When writing tests
+//
+// Make sure every series of writes to the remote has a
+// fstest.CheckItems() before use. This make sure the directory
+// listing is now consistent and stops cascading errors.
+//
+// Call accounting.GlobalStats().ResetCounters() before every fs.Sync() as it
+// uses the error count internally.
+
+package operations_test
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "regexp"
+ "strings"
+ "testing"
+ "time"
+
+ _ "github.com/mewsen/rclone-studip-backend-oot/backend/studip"
+ _ "github.com/rclone/rclone/backend/all" // import all backends
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/filter"
+ "github.com/rclone/rclone/fs/fshttp"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/fstest"
+ "github.com/rclone/rclone/fstest/fstests"
+ "github.com/rclone/rclone/lib/pacer"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "golang.org/x/text/cases"
+ "golang.org/x/text/language"
+)
+
+// Some times used in the tests
+var (
+ t1 = fstest.Time("2001-02-03T04:05:06.499999999Z")
+ t2 = fstest.Time("2011-12-25T12:59:59.123456789Z")
+ t3 = fstest.Time("2011-12-30T12:59:59.000000000Z")
+)
+
+// TestMain drives the tests
+func TestMain(m *testing.M) {
+ fstest.TestMain(m)
+}
+
+func TestMkdir(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+
+ err := operations.Mkdir(ctx, r.Fremote, "")
+ require.NoError(t, err)
+ fstest.CheckListing(t, r.Fremote, []fstest.Item{})
+
+ err = operations.Mkdir(ctx, r.Fremote, "")
+ require.NoError(t, err)
+}
+
+func TestLsd(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1)
+
+ r.CheckRemoteItems(t, file1)
+
+ var buf bytes.Buffer
+ err := operations.ListDir(ctx, r.Fremote, &buf)
+ require.NoError(t, err)
+ res := buf.String()
+ assert.Contains(t, res, "sub dir\n")
+}
+
+func TestLs(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
+ file2 := r.WriteBoth(ctx, "empty space", "-", t2)
+
+ r.CheckRemoteItems(t, file1, file2)
+
+ var buf bytes.Buffer
+ err := operations.List(ctx, r.Fremote, &buf)
+ require.NoError(t, err)
+ res := buf.String()
+ assert.Contains(t, res, " 1 empty space\n")
+ assert.Contains(t, res, " 60 potato2\n")
+}
+
+func TestLsWithFilesFrom(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+ file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
+ file2 := r.WriteBoth(ctx, "empty space", "-", t2)
+
+ r.CheckRemoteItems(t, file1, file2)
+
+ // Set the --files-from equivalent
+ f, err := filter.NewFilter(nil)
+ require.NoError(t, err)
+ require.NoError(t, f.AddFile("potato2"))
+ require.NoError(t, f.AddFile("notfound"))
+
+ // Change the active filter
+ ctx = filter.ReplaceConfig(ctx, f)
+
+ var buf bytes.Buffer
+ err = operations.List(ctx, r.Fremote, &buf)
+ require.NoError(t, err)
+ assert.Equal(t, " 60 potato2\n", buf.String())
+
+ // Now try with --no-traverse
+ ci.NoTraverse = true
+
+ buf.Reset()
+ err = operations.List(ctx, r.Fremote, &buf)
+ require.NoError(t, err)
+ assert.Equal(t, " 60 potato2\n", buf.String())
+}
+
+func TestLsLong(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
+ file2 := r.WriteBoth(ctx, "empty space", "-", t2)
+
+ r.CheckRemoteItems(t, file1, file2)
+
+ var buf bytes.Buffer
+ err := operations.ListLong(ctx, r.Fremote, &buf)
+ require.NoError(t, err)
+ res := buf.String()
+ lines := strings.Split(strings.Trim(res, "\n"), "\n")
+ assert.Equal(t, 2, len(lines))
+
+ timeFormat := "2006-01-02 15:04:05.000000000"
+ precision := r.Fremote.Precision()
+ location := time.Now().Location()
+ checkTime := func(m, filename string, expected time.Time) {
+ modTime, err := time.ParseInLocation(timeFormat, m, location) // parse as localtime
+ if err != nil {
+ t.Errorf("Error parsing %q: %v", m, err)
+ } else {
+ fstest.AssertTimeEqualWithPrecision(t, filename, expected, modTime, precision)
+ }
+ }
+
+ m1 := regexp.MustCompile(`(?m)^ 1 (\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{9}) empty space$`)
+ if ms := m1.FindStringSubmatch(res); ms == nil {
+ t.Errorf("empty space missing: %q", res)
+ } else {
+ checkTime(ms[1], "empty space", t2.Local())
+ }
+
+ m2 := regexp.MustCompile(`(?m)^ 60 (\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{9}) potato2$`)
+ if ms := m2.FindStringSubmatch(res); ms == nil {
+ t.Errorf("potato2 missing: %q", res)
+ } else {
+ checkTime(ms[1], "potato2", t1.Local())
+ }
+}
+
+func TestHashSums(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
+ file2 := r.WriteBoth(ctx, "empty space", "-", t2)
+
+ r.CheckRemoteItems(t, file1, file2)
+
+ hashes := r.Fremote.Hashes()
+
+ var quickXorHash hash.Type
+ err := quickXorHash.Set("QuickXorHash")
+ require.NoError(t, err)
+
+ for _, test := range []struct {
+ name string
+ download bool
+ base64 bool
+ ht hash.Type
+ want []string
+ }{
+ {
+ ht: hash.MD5,
+ want: []string{
+ "336d5ebc5436534e61d16e63ddfca327 empty space\n",
+ "d6548b156ea68a4e003e786df99eee76 potato2\n",
+ },
+ },
+ {
+ ht: hash.MD5,
+ download: true,
+ want: []string{
+ "336d5ebc5436534e61d16e63ddfca327 empty space\n",
+ "d6548b156ea68a4e003e786df99eee76 potato2\n",
+ },
+ },
+ {
+ ht: hash.SHA1,
+ want: []string{
+ "3bc15c8aae3e4124dd409035f32ea2fd6835efc9 empty space\n",
+ "9dc7f7d3279715991a22853f5981df582b7f9f6d potato2\n",
+ },
+ },
+ {
+ ht: hash.SHA1,
+ download: true,
+ want: []string{
+ "3bc15c8aae3e4124dd409035f32ea2fd6835efc9 empty space\n",
+ "9dc7f7d3279715991a22853f5981df582b7f9f6d potato2\n",
+ },
+ },
+ {
+ ht: quickXorHash,
+ want: []string{
+ "2d00000000000000000000000100000000000000 empty space\n",
+ "4001dad296b6b4a52d6d694b67dad296b6b4a52d potato2\n",
+ },
+ },
+ {
+ ht: quickXorHash,
+ download: true,
+ want: []string{
+ "2d00000000000000000000000100000000000000 empty space\n",
+ "4001dad296b6b4a52d6d694b67dad296b6b4a52d potato2\n",
+ },
+ },
+ {
+ ht: quickXorHash,
+ base64: true,
+ want: []string{
+ "LQAAAAAAAAAAAAAAAQAAAAAAAAA= empty space\n",
+ "QAHa0pa2tKUtbWlLZ9rSlra0pS0= potato2\n",
+ },
+ },
+ {
+ ht: quickXorHash,
+ base64: true,
+ download: true,
+ want: []string{
+ "LQAAAAAAAAAAAAAAAQAAAAAAAAA= empty space\n",
+ "QAHa0pa2tKUtbWlLZ9rSlra0pS0= potato2\n",
+ },
+ },
+ } {
+ if !hashes.Contains(test.ht) {
+ continue
+ }
+ name := cases.Title(language.Und, cases.NoLower).String(test.ht.String())
+ if test.download {
+ name += "Download"
+ }
+ if test.base64 {
+ name += "Base64"
+ }
+ t.Run(name, func(t *testing.T) {
+ var buf bytes.Buffer
+ err := operations.HashLister(ctx, test.ht, test.base64, test.download, r.Fremote, &buf)
+ require.NoError(t, err)
+ res := buf.String()
+ for _, line := range test.want {
+ assert.Contains(t, res, line)
+ }
+ })
+ }
+}
+
+func TestHashSumsWithErrors(t *testing.T) {
+ ctx := context.Background()
+ memFs, err := fs.NewFs(ctx, ":memory:")
+ require.NoError(t, err)
+
+ // Make a test file
+ content := "-"
+ item1 := fstest.NewItem("file1", content, t1)
+ _ = fstests.PutTestContents(ctx, t, memFs, &item1, content, true)
+
+ // MemoryFS supports MD5
+ buf := &bytes.Buffer{}
+ err = operations.HashLister(ctx, hash.MD5, false, false, memFs, buf)
+ require.NoError(t, err)
+ assert.Contains(t, buf.String(), "336d5ebc5436534e61d16e63ddfca327 file1\n")
+
+ // MemoryFS can't do SHA1, but UNSUPPORTED must not appear in the output
+ buf.Reset()
+ err = operations.HashLister(ctx, hash.SHA1, false, false, memFs, buf)
+ require.NoError(t, err)
+ assert.NotContains(t, buf.String(), " UNSUPPORTED ")
+
+ // ERROR must not appear in the output either
+ assert.NotContains(t, buf.String(), " ERROR ")
+ // TODO mock an unreadable file
+}
+
+func TestHashStream(t *testing.T) {
+ reader := strings.NewReader("")
+ in := io.NopCloser(reader)
+ out := &bytes.Buffer{}
+ for _, test := range []struct {
+ input string
+ ht hash.Type
+ wantHex string
+ wantBase64 string
+ }{
+ {
+ input: "",
+ ht: hash.MD5,
+ wantHex: "d41d8cd98f00b204e9800998ecf8427e -\n",
+ wantBase64: "1B2M2Y8AsgTpgAmY7PhCfg== -\n",
+ },
+ {
+ input: "",
+ ht: hash.SHA1,
+ wantHex: "da39a3ee5e6b4b0d3255bfef95601890afd80709 -\n",
+ wantBase64: "2jmj7l5rSw0yVb_vlWAYkK_YBwk= -\n",
+ },
+ {
+ input: "Hello world!",
+ ht: hash.MD5,
+ wantHex: "86fb269d190d2c85f6e0468ceca42a20 -\n",
+ wantBase64: "hvsmnRkNLIX24EaM7KQqIA== -\n",
+ },
+ {
+ input: "Hello world!",
+ ht: hash.SHA1,
+ wantHex: "d3486ae9136e7856bc42212385ea797094475802 -\n",
+ wantBase64: "00hq6RNueFa8QiEjhep5cJRHWAI= -\n",
+ },
+ } {
+ reader.Reset(test.input)
+ require.NoError(t, operations.HashSumStream(test.ht, false, in, out))
+ assert.Equal(t, test.wantHex, out.String())
+ _, _ = reader.Seek(0, io.SeekStart)
+ out.Reset()
+ require.NoError(t, operations.HashSumStream(test.ht, true, in, out))
+ assert.Equal(t, test.wantBase64, out.String())
+ out.Reset()
+ }
+}
+
+func TestSuffixName(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ for _, test := range []struct {
+ remote string
+ suffix string
+ keepExt bool
+ want string
+ }{
+ {"test.txt", "", false, "test.txt"},
+ {"test.txt", "", true, "test.txt"},
+ {"test.txt", "-suffix", false, "test.txt-suffix"},
+ {"test.txt", "-suffix", true, "test-suffix.txt"},
+ {"test.txt.csv", "-suffix", false, "test.txt.csv-suffix"},
+ {"test.txt.csv", "-suffix", true, "test-suffix.txt.csv"},
+ {"test", "-suffix", false, "test-suffix"},
+ {"test", "-suffix", true, "test-suffix"},
+ {"test.html", "-suffix", true, "test-suffix.html"},
+ {"test.html.txt", "-suffix", true, "test-suffix.html.txt"},
+ {"test.csv.html.txt", "-suffix", true, "test-suffix.csv.html.txt"},
+ {"test.badext.csv.html.txt", "-suffix", true, "test.badext-suffix.csv.html.txt"},
+ {"test.badext", "-suffix", true, "test-suffix.badext"},
+ } {
+ ci.Suffix = test.suffix
+ ci.SuffixKeepExtension = test.keepExt
+ got := operations.SuffixName(ctx, test.remote)
+ assert.Equal(t, test.want, got, fmt.Sprintf("%+v", test))
+ }
+}
+
+func TestCount(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+ file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
+ file2 := r.WriteBoth(ctx, "empty space", "-", t2)
+ file3 := r.WriteBoth(ctx, "sub dir/potato3", "hello", t2)
+
+ r.CheckRemoteItems(t, file1, file2, file3)
+
+ // Check the MaxDepth too
+ ci.MaxDepth = 1
+
+ objects, size, sizeless, err := operations.Count(ctx, r.Fremote)
+ require.NoError(t, err)
+ assert.Equal(t, int64(2), objects)
+ assert.Equal(t, int64(61), size)
+ assert.Equal(t, int64(0), sizeless)
+}
+
+func TestDelete(t *testing.T) {
+ ctx := context.Background()
+ fi, err := filter.NewFilter(nil)
+ require.NoError(t, err)
+ fi.Opt.MaxSize = 60
+ ctx = filter.ReplaceConfig(ctx, fi)
+ r := fstest.NewRun(t)
+ file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes
+ file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes
+ file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
+ r.CheckRemoteItems(t, file1, file2, file3)
+
+ err = operations.Delete(ctx, r.Fremote)
+ require.NoError(t, err)
+ r.CheckRemoteItems(t, file3)
+}
+
+func isChunker(f fs.Fs) bool {
+ return strings.HasPrefix(f.Name(), "TestChunker")
+}
+
+func skipIfChunker(t *testing.T, f fs.Fs) {
+ if isChunker(f) {
+ t.Skip("Skipping test on chunker backend")
+ }
+}
+
+func TestMaxDelete(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+ accounting.GlobalStats().ResetCounters()
+ ci.MaxDelete = 2
+ defer r.Finalise()
+ skipIfChunker(t, r.Fremote) // chunker does copy/delete on s3
+ file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes
+ file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes
+ file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
+ r.CheckRemoteItems(t, file1, file2, file3)
+ err := operations.Delete(ctx, r.Fremote)
+
+ require.Error(t, err)
+ objects, _, _, err := operations.Count(ctx, r.Fremote)
+ require.NoError(t, err)
+ assert.Equal(t, int64(1), objects)
+}
+
+// TestMaxDeleteSizeLargeFile one of the files is larger than allowed
+func TestMaxDeleteSizeLargeFile(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+ accounting.GlobalStats().ResetCounters()
+ ci.MaxDeleteSize = 70
+ defer r.Finalise()
+ skipIfChunker(t, r.Fremote) // chunker does copy/delete on s3
+ file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes
+ file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes
+ file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
+ r.CheckRemoteItems(t, file1, file2, file3)
+
+ err := operations.Delete(ctx, r.Fremote)
+ require.Error(t, err)
+ r.CheckRemoteItems(t, file3)
+}
+
+func TestMaxDeleteSize(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+ accounting.GlobalStats().ResetCounters()
+ ci.MaxDeleteSize = 160
+ defer r.Finalise()
+ skipIfChunker(t, r.Fremote) // chunker does copy/delete on s3
+ file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes
+ file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes
+ file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
+ r.CheckRemoteItems(t, file1, file2, file3)
+
+ err := operations.Delete(ctx, r.Fremote)
+ require.Error(t, err)
+ objects, _, _, err := operations.Count(ctx, r.Fremote)
+ require.NoError(t, err)
+ assert.Equal(t, int64(1), objects) // 10 or 100 bytes
+}
+
+func TestReadFile(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ defer r.Finalise()
+
+ contents := "A file to read the contents."
+ file := r.WriteObject(ctx, "ReadFile", contents, t1)
+ r.CheckRemoteItems(t, file)
+
+ o, err := r.Fremote.NewObject(ctx, file.Path)
+ require.NoError(t, err)
+
+ buf, err := operations.ReadFile(ctx, o)
+ require.NoError(t, err)
+ assert.Equal(t, contents, string(buf))
+}
+
+func TestRetry(t *testing.T) {
+ ctx := context.Background()
+
+ var i int
+ var err error
+ fn := func() error {
+ i--
+ if i <= 0 {
+ return nil
+ }
+ return err
+ }
+
+ i, err = 3, fmt.Errorf("Wrapped EOF is retriable: %w", io.EOF)
+ assert.Equal(t, nil, operations.Retry(ctx, nil, 5, fn))
+ assert.Equal(t, 0, i)
+
+ i, err = 10, pacer.RetryAfterError(errors.New("BANG"), 10*time.Millisecond)
+ assert.Equal(t, err, operations.Retry(ctx, nil, 5, fn))
+ assert.Equal(t, 5, i)
+
+ i, err = 10, fs.ErrorObjectNotFound
+ assert.Equal(t, fs.ErrorObjectNotFound, operations.Retry(ctx, nil, 5, fn))
+ assert.Equal(t, 9, i)
+
+}
+
+func TestCat(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ file1 := r.WriteBoth(ctx, "file1", "ABCDEFGHIJ", t1)
+ file2 := r.WriteBoth(ctx, "file2", "012345678", t2)
+
+ r.CheckRemoteItems(t, file1, file2)
+
+ for _, test := range []struct {
+ offset int64
+ count int64
+ separator string
+ a string
+ b string
+ }{
+ {0, -1, "", "ABCDEFGHIJ", "012345678"},
+ {0, 5, "", "ABCDE", "01234"},
+ {-3, -1, "", "HIJ", "678"},
+ {1, 3, "", "BCD", "123"},
+ {0, -1, "\n", "ABCDEFGHIJ", "012345678"},
+ } {
+ var buf bytes.Buffer
+ err := operations.Cat(ctx, r.Fremote, &buf, test.offset, test.count, []byte(test.separator))
+ require.NoError(t, err)
+ res := buf.String()
+
+ if res != test.a+test.separator+test.b+test.separator && res != test.b+test.separator+test.a+test.separator {
+ t.Errorf("Incorrect output from Cat(%d,%d,%s): %q", test.offset, test.count, test.separator, res)
+ }
+ }
+}
+
+func TestPurge(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRunIndividual(t) // make new container (azureblob has delayed mkdir after rmdir)
+ r.Mkdir(ctx, r.Fremote)
+
+ // Make some files and dirs
+ r.ForceMkdir(ctx, r.Fremote)
+ file1 := r.WriteObject(ctx, "A1/B1/C1/one", "aaa", t1)
+ //..and dirs we expect to delete
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A2"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2/C2"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C3"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3/B3"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3/B3/C4"))
+ //..and one more file at the end
+ file2 := r.WriteObject(ctx, "A1/two", "bbb", t2)
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ []fstest.Item{
+ file1, file2,
+ },
+ []string{
+ "A1",
+ "A1/B1",
+ "A1/B1/C1",
+ "A2",
+ "A1/B2",
+ "A1/B2/C2",
+ "A1/B1/C3",
+ "A3",
+ "A3/B3",
+ "A3/B3/C4",
+ },
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+
+ require.NoError(t, operations.Purge(ctx, r.Fremote, "A1/B1"))
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ []fstest.Item{
+ file2,
+ },
+ []string{
+ "A1",
+ "A2",
+ "A1/B2",
+ "A1/B2/C2",
+ "A3",
+ "A3/B3",
+ "A3/B3/C4",
+ },
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+
+ require.NoError(t, operations.Purge(ctx, r.Fremote, ""))
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ []fstest.Item{},
+ []string{},
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+
+}
+
+func TestRmdirsNoLeaveRoot(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ r.Mkdir(ctx, r.Fremote)
+
+ // Make some files and dirs we expect to keep
+ r.ForceMkdir(ctx, r.Fremote)
+ file1 := r.WriteObject(ctx, "A1/B1/C1/one", "aaa", t1)
+ //..and dirs we expect to delete
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A2"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2/C2"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C3"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3/B3"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3/B3/C4"))
+ //..and one more file at the end
+ file2 := r.WriteObject(ctx, "A1/two", "bbb", t2)
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ []fstest.Item{
+ file1, file2,
+ },
+ []string{
+ "A1",
+ "A1/B1",
+ "A1/B1/C1",
+ "A2",
+ "A1/B2",
+ "A1/B2/C2",
+ "A1/B1/C3",
+ "A3",
+ "A3/B3",
+ "A3/B3/C4",
+ },
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+
+ require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "A3/B3/C4", false))
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ []fstest.Item{
+ file1, file2,
+ },
+ []string{
+ "A1",
+ "A1/B1",
+ "A1/B1/C1",
+ "A2",
+ "A1/B2",
+ "A1/B2/C2",
+ "A1/B1/C3",
+ "A3",
+ "A3/B3",
+ },
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+
+ require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "", false))
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ []fstest.Item{
+ file1, file2,
+ },
+ []string{
+ "A1",
+ "A1/B1",
+ "A1/B1/C1",
+ },
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+
+ // Delete the files so we can remove everything including the root
+ for _, file := range []fstest.Item{file1, file2} {
+ o, err := r.Fremote.NewObject(ctx, file.Path)
+ require.NoError(t, err)
+ require.NoError(t, o.Remove(ctx))
+ }
+
+ require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "", false))
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ []fstest.Item{},
+ []string{},
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+}
+
+func TestRmdirsLeaveRoot(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ r.Mkdir(ctx, r.Fremote)
+
+ r.ForceMkdir(ctx, r.Fremote)
+
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C1"))
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ []fstest.Item{},
+ []string{
+ "A1",
+ "A1/B1",
+ "A1/B1/C1",
+ },
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+
+ require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "A1", true))
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ []fstest.Item{},
+ []string{
+ "A1",
+ },
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+}
+
+func TestRmdirsWithFilter(t *testing.T) {
+ ctx := context.Background()
+ ctx, fi := filter.AddConfig(ctx)
+ require.NoError(t, fi.AddRule("+ /A1/B1/**"))
+ require.NoError(t, fi.AddRule("- *"))
+ r := fstest.NewRun(t)
+ r.Mkdir(ctx, r.Fremote)
+
+ r.ForceMkdir(ctx, r.Fremote)
+
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C1"))
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ []fstest.Item{},
+ []string{
+ "A1",
+ "A1/B1",
+ "A1/B1/C1",
+ },
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+
+ require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "", false))
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ []fstest.Item{},
+ []string{
+ "A1",
+ },
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+}
+
+func TestCopyURL(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+
+ contents := "file contents\n"
+ file1 := r.WriteFile("file1", contents, t1)
+ file2 := r.WriteFile("file2", contents, t1)
+ r.Mkdir(ctx, r.Fremote)
+ r.CheckRemoteItems(t)
+
+ // check when reading from regular HTTP server
+ status := 0
+ nameHeader := false
+ headerFilename := "headerfilename.txt"
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if status != 0 {
+ http.Error(w, "an error occurred", status)
+ }
+ if nameHeader {
+ w.Header().Set("Content-Disposition", `attachment; filename="folder\`+headerFilename+`"`)
+ }
+ _, err := w.Write([]byte(contents))
+ assert.NoError(t, err)
+ })
+ ts := httptest.NewServer(handler)
+ defer ts.Close()
+
+ o, err := operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, false)
+ require.NoError(t, err)
+ assert.Equal(t, int64(len(contents)), o.Size())
+
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported)
+
+ // Check file clobbering
+ _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, true)
+ require.Error(t, err)
+
+ // Check auto file naming
+ status = 0
+ urlFileName := "filename.txt"
+ o, err = operations.CopyURL(ctx, r.Fremote, "", ts.URL+"/"+urlFileName, true, false, false)
+ require.NoError(t, err)
+ assert.Equal(t, int64(len(contents)), o.Size())
+ assert.Equal(t, urlFileName, o.Remote())
+
+ // Check header file naming
+ nameHeader = true
+ o, err = operations.CopyURL(ctx, r.Fremote, "", ts.URL, true, true, false)
+ require.NoError(t, err)
+ assert.Equal(t, int64(len(contents)), o.Size())
+ assert.Equal(t, headerFilename, o.Remote())
+
+ // Check auto file naming when url without file name
+ _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, true, false, false)
+ require.Error(t, err)
+
+ // Check header file naming without header set
+ nameHeader = false
+ _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, true, true, false)
+ require.Error(t, err)
+
+ // Check an error is returned for a 404
+ status = http.StatusNotFound
+ o, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, false)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "Not Found")
+ assert.Nil(t, o)
+ status = 0
+
+ // check when reading from unverified HTTPS server
+ ci.InsecureSkipVerify = true
+ fshttp.ResetTransport()
+ defer fshttp.ResetTransport()
+ tss := httptest.NewTLSServer(handler)
+ defer tss.Close()
+
+ o, err = operations.CopyURL(ctx, r.Fremote, "file2", tss.URL, false, false, false)
+ require.NoError(t, err)
+ assert.Equal(t, int64(len(contents)), o.Size())
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1), fstest.NewItem(headerFilename, contents, t1)}, nil, fs.ModTimeNotSupported)
+}
+
+func TestCopyURLToWriter(t *testing.T) {
+ ctx := context.Background()
+ contents := "file contents\n"
+
+ // check when reading from regular HTTP server
+ status := 0
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if status != 0 {
+ http.Error(w, "an error occurred", status)
+ return
+ }
+ _, err := w.Write([]byte(contents))
+ assert.NoError(t, err)
+ })
+ ts := httptest.NewServer(handler)
+ defer ts.Close()
+
+ // test normal fetch
+ var buf bytes.Buffer
+ err := operations.CopyURLToWriter(ctx, ts.URL, &buf)
+ require.NoError(t, err)
+ assert.Equal(t, contents, buf.String())
+
+ // test fetch with error
+ status = http.StatusNotFound
+ buf.Reset()
+ err = operations.CopyURLToWriter(ctx, ts.URL, &buf)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "Not Found")
+ assert.Equal(t, 0, len(buf.String()))
+}
+
+func TestMoveFile(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+
+ file1 := r.WriteFile("file1", "file1 contents", t1)
+ r.CheckLocalItems(t, file1)
+
+ file2 := file1
+ file2.Path = "sub/file2"
+
+ err := operations.MoveFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t)
+ r.CheckRemoteItems(t, file2)
+
+ r.WriteFile("file1", "file1 contents", t1)
+ r.CheckLocalItems(t, file1)
+
+ err = operations.MoveFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t)
+ r.CheckRemoteItems(t, file2)
+
+ err = operations.MoveFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t)
+ r.CheckRemoteItems(t, file2)
+}
+
+func TestMoveFileWithIgnoreExisting(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+ file1 := r.WriteFile("file1", "file1 contents", t1)
+ r.CheckLocalItems(t, file1)
+
+ ci.IgnoreExisting = true
+
+ err := operations.MoveFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t)
+ r.CheckRemoteItems(t, file1)
+
+ // Recreate file with updated content
+ file1b := r.WriteFile("file1", "file1 modified", t2)
+ r.CheckLocalItems(t, file1b)
+
+ // Ensure modified file did not transfer and was not deleted
+ err = operations.MoveFile(ctx, r.Fremote, r.Flocal, file1.Path, file1b.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t, file1b)
+ r.CheckRemoteItems(t, file1)
+}
+
+func TestCaseInsensitiveMoveFile(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ if !r.Fremote.Features().CaseInsensitive {
+ return
+ }
+
+ file1 := r.WriteFile("file1", "file1 contents", t1)
+ r.CheckLocalItems(t, file1)
+
+ file2 := file1
+ file2.Path = "sub/file2"
+
+ err := operations.MoveFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t)
+ r.CheckRemoteItems(t, file2)
+
+ r.WriteFile("file1", "file1 contents", t1)
+ r.CheckLocalItems(t, file1)
+
+ err = operations.MoveFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t)
+ r.CheckRemoteItems(t, file2)
+
+ file2Capitalized := file2
+ file2Capitalized.Path = "sub/File2"
+
+ err = operations.MoveFile(ctx, r.Fremote, r.Fremote, file2Capitalized.Path, file2.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t)
+ r.CheckRemoteItems(t, file2Capitalized)
+}
+
+func TestCaseInsensitiveMoveFileDryRun(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+ if !r.Fremote.Features().CaseInsensitive {
+ return
+ }
+
+ file1 := r.WriteObject(ctx, "hello", "world", t1)
+ r.CheckRemoteItems(t, file1)
+
+ ci.DryRun = true
+ err := operations.MoveFile(ctx, r.Fremote, r.Fremote, "HELLO", file1.Path)
+ require.NoError(t, err)
+ r.CheckRemoteItems(t, file1)
+}
+
+func TestMoveFileBackupDir(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ r := fstest.NewRun(t)
+ if !operations.CanServerSideMove(r.Fremote) {
+ t.Skip("Skipping test as remote does not support server-side move or copy")
+ }
+
+ ci.BackupDir = r.FremoteName + "/backup"
+
+ file1 := r.WriteFile("dst/file1", "file1 contents", t1)
+ r.CheckLocalItems(t, file1)
+
+ file1old := r.WriteObject(ctx, "dst/file1", "file1 contents old", t1)
+ r.CheckRemoteItems(t, file1old)
+
+ err := operations.MoveFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
+ require.NoError(t, err)
+ r.CheckLocalItems(t)
+ file1old.Path = "backup/dst/file1"
+ r.CheckRemoteItems(t, file1old, file1)
+}
+
+// testFsInfo is for unit testing fs.Info
+type testFsInfo struct {
+ name string
+ root string
+ stringVal string
+ precision time.Duration
+ hashes hash.Set
+ features fs.Features
+}
+
+// Name of the remote (as passed into NewFs)
+func (i *testFsInfo) Name() string { return i.name }
+
+// Root of the remote (as passed into NewFs)
+func (i *testFsInfo) Root() string { return i.root }
+
+// String returns a description of the FS
+func (i *testFsInfo) String() string { return i.stringVal }
+
+// Precision of the ModTimes in this Fs
+func (i *testFsInfo) Precision() time.Duration { return i.precision }
+
+// Returns the supported hash types of the filesystem
+func (i *testFsInfo) Hashes() hash.Set { return i.hashes }
+
+// Returns the supported hash types of the filesystem
+func (i *testFsInfo) Features() *fs.Features { return &i.features }
+
+func TestSameConfig(t *testing.T) {
+ a := &testFsInfo{name: "name", root: "root"}
+ for _, test := range []struct {
+ name string
+ root string
+ expected bool
+ }{
+ {"name", "root", true},
+ {"name", "rooty", true},
+ {"namey", "root", false},
+ {"namey", "roott", false},
+ } {
+ b := &testFsInfo{name: test.name, root: test.root}
+ actual := operations.SameConfig(a, b)
+ assert.Equal(t, test.expected, actual)
+ actual = operations.SameConfig(b, a)
+ assert.Equal(t, test.expected, actual)
+ }
+}
+
+func TestSame(t *testing.T) {
+ a := &testFsInfo{name: "name", root: "root"}
+ for _, test := range []struct {
+ name string
+ root string
+ expected bool
+ }{
+ {"name", "root", true},
+ {"name", "rooty", false},
+ {"namey", "root", false},
+ {"namey", "roott", false},
+ } {
+ b := &testFsInfo{name: test.name, root: test.root}
+ actual := operations.Same(a, b)
+ assert.Equal(t, test.expected, actual)
+ actual = operations.Same(b, a)
+ assert.Equal(t, test.expected, actual)
+ }
+}
+
+// testFs is for unit testing fs.Fs
+type testFs struct {
+ testFsInfo
+}
+
+func (i *testFs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+ return nil, nil
+}
+
+func (i *testFs) NewObject(ctx context.Context, remote string) (fs.Object, error) { return nil, nil }
+
+func (i *testFs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+ return nil, nil
+}
+
+func (i *testFs) Mkdir(ctx context.Context, dir string) error { return nil }
+
+func (i *testFs) Rmdir(ctx context.Context, dir string) error { return nil }
+
+// copied from TestOverlapping because the behavior of OverlappingFilterCheck should be identical to Overlapping
+// when no filters are set
+func TestOverlappingFilterCheckWithoutFilter(t *testing.T) {
+ ctx := context.Background()
+ src := &testFs{testFsInfo{name: "name", root: "root"}}
+ slash := string(os.PathSeparator) // native path separator
+ for _, test := range []struct {
+ name string
+ root string
+ expected bool
+ }{
+ {"name", "root", true},
+ {"name", "/root", true},
+ {"namey", "root", false},
+ {"name", "rooty", false},
+ {"namey", "rooty", false},
+ {"name", "roo", false},
+ {"name", "root/toot", true},
+ {"name", "root/toot/", true},
+ {"name", "root" + slash + "toot", true},
+ {"name", "root" + slash + "toot" + slash, true},
+ {"name", "", true},
+ {"name", "/", true},
+ } {
+ dst := &testFs{testFsInfo{name: test.name, root: test.root}}
+ what := fmt.Sprintf("(%q,%q) vs (%q,%q)", src.name, src.root, dst.name, dst.root)
+ actual := operations.OverlappingFilterCheck(ctx, src, dst)
+ assert.Equal(t, test.expected, actual, what)
+ actual = operations.OverlappingFilterCheck(ctx, dst, src)
+ assert.Equal(t, test.expected, actual, what)
+ }
+}
+
+func TestOverlappingFilterCheckWithFilter(t *testing.T) {
+ ctx := context.Background()
+ fi, err := filter.NewFilter(nil)
+ require.NoError(t, err)
+ require.NoError(t, fi.Add(false, "/exclude/"))
+ require.NoError(t, fi.Add(false, "/Exclude2/"))
+ require.NoError(t, fi.Add(true, "*"))
+ ctx = filter.ReplaceConfig(ctx, fi)
+
+ src := &testFs{testFsInfo{name: "name", root: "root"}}
+ src.features.CaseInsensitive = true
+ slash := string(os.PathSeparator) // native path separator
+ for _, test := range []struct {
+ name string
+ root string
+ expected bool
+ }{
+ {"name", "root", true},
+ {"name", "ROOT", true}, // case insensitive is set
+ {"name", "/root", true},
+ {"name", "root/", true},
+ {"name", "root" + slash, true},
+ {"name", "root/exclude", false},
+ {"name", "root/Exclude2", false},
+ {"name", "root/include", true},
+ {"name", "root/exclude/", false},
+ {"name", "root/Exclude2/", false},
+ {"name", "root/exclude/sub", false},
+ {"name", "root/Exclude2/sub", false},
+ {"name", "/root/exclude/", false},
+ {"name", "root" + slash + "exclude", false},
+ {"name", "root" + slash + "exclude" + slash, false},
+ {"namey", "root/include", false},
+ {"namey", "root/include/", false},
+ {"namey", "root" + slash + "include", false},
+ {"namey", "root" + slash + "include" + slash, false},
+ } {
+ dst := &testFs{testFsInfo{name: test.name, root: test.root}}
+ dst.features.CaseInsensitive = true
+ what := fmt.Sprintf("(%q,%q) vs (%q,%q)", src.name, src.root, dst.name, dst.root)
+ actual := operations.OverlappingFilterCheck(ctx, dst, src)
+ assert.Equal(t, test.expected, actual, what)
+ actual = operations.OverlappingFilterCheck(ctx, src, dst)
+ assert.Equal(t, test.expected, actual, what)
+ }
+}
+
+func TestListFormat(t *testing.T) {
+ item0 := &operations.ListJSONItem{
+ Path: "a",
+ Name: "a",
+ Encrypted: "encryptedFileName",
+ Size: 1,
+ MimeType: "application/octet-stream",
+ ModTime: operations.Timestamp{
+ When: t1,
+ Format: "2006-01-02T15:04:05.000000000Z07:00"},
+ IsDir: false,
+ Hashes: map[string]string{
+ "md5": "0cc175b9c0f1b6a831c399e269772661",
+ "sha1": "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8",
+ "dropbox": "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8",
+ "quickxor": "6100000000000000000000000100000000000000"},
+ ID: "fileID",
+ OrigID: "fileOrigID",
+ }
+
+ item1 := &operations.ListJSONItem{
+ Path: "subdir",
+ Name: "subdir",
+ Encrypted: "encryptedDirName",
+ Size: -1,
+ MimeType: "inode/directory",
+ ModTime: operations.Timestamp{
+ When: t2,
+ Format: "2006-01-02T15:04:05.000000000Z07:00"},
+ IsDir: true,
+ Hashes: map[string]string(nil),
+ ID: "dirID",
+ OrigID: "dirOrigID",
+ }
+
+ var list operations.ListFormat
+ list.AddPath()
+ list.SetDirSlash(false)
+ assert.Equal(t, "subdir", list.Format(item1))
+
+ list.SetDirSlash(true)
+ assert.Equal(t, "subdir/", list.Format(item1))
+
+ list.SetOutput(nil)
+ assert.Equal(t, "", list.Format(item1))
+
+ list.AppendOutput(func(item *operations.ListJSONItem) string { return "a" })
+ list.AppendOutput(func(item *operations.ListJSONItem) string { return "b" })
+ assert.Equal(t, "ab", list.Format(item1))
+ list.SetSeparator(":::")
+ assert.Equal(t, "a:::b", list.Format(item1))
+
+ list.SetOutput(nil)
+ list.AddModTime("")
+ assert.Equal(t, t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0))
+
+ list.SetOutput(nil)
+ list.AddModTime("unix")
+ assert.Equal(t, fmt.Sprint(t1.Local().Unix()), list.Format(item0))
+
+ list.SetOutput(nil)
+ list.AddModTime("unixnano")
+ assert.Equal(t, fmt.Sprint(t1.Local().UnixNano()), list.Format(item0))
+
+ list.SetOutput(nil)
+ list.SetSeparator("|")
+ list.AddID()
+ list.AddOrigID()
+ assert.Equal(t, "fileID|fileOrigID", list.Format(item0))
+ assert.Equal(t, "dirID|dirOrigID", list.Format(item1))
+
+ list.SetOutput(nil)
+ list.AddMimeType()
+ assert.Contains(t, list.Format(item0), "/")
+ assert.Equal(t, "inode/directory", list.Format(item1))
+
+ list.SetOutput(nil)
+ list.AddMetadata()
+ assert.Equal(t, "{}", list.Format(item0))
+ assert.Equal(t, "{}", list.Format(item1))
+
+ list.SetOutput(nil)
+ list.AddPath()
+ list.SetAbsolute(true)
+ assert.Equal(t, "/a", list.Format(item0))
+ list.SetAbsolute(false)
+ assert.Equal(t, "a", list.Format(item0))
+
+ list.SetOutput(nil)
+ list.AddSize()
+ assert.Equal(t, "1", list.Format(item0))
+
+ list.AddPath()
+ list.AddModTime("")
+ list.SetDirSlash(true)
+ list.SetSeparator("__SEP__")
+ assert.Equal(t, "1__SEP__a__SEP__"+t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0))
+ assert.Equal(t, "-1__SEP__subdir/__SEP__"+t2.Local().Format("2006-01-02 15:04:05"), list.Format(item1))
+
+ for _, test := range []struct {
+ ht hash.Type
+ want string
+ }{
+ {hash.MD5, "0cc175b9c0f1b6a831c399e269772661"},
+ {hash.SHA1, "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8"},
+ } {
+ list.SetOutput(nil)
+ list.AddHash(test.ht)
+ assert.Equal(t, test.want, list.Format(item0))
+ }
+
+ list.SetOutput(nil)
+ list.SetSeparator("|")
+ list.SetCSV(true)
+ list.AddSize()
+ list.AddPath()
+ list.AddModTime("")
+ list.SetDirSlash(true)
+ assert.Equal(t, "1|a|"+t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0))
+ assert.Equal(t, "-1|subdir/|"+t2.Local().Format("2006-01-02 15:04:05"), list.Format(item1))
+
+ list.SetOutput(nil)
+ list.SetSeparator("|")
+ list.AddPath()
+ list.AddEncrypted()
+ assert.Equal(t, "a|encryptedFileName", list.Format(item0))
+ assert.Equal(t, "subdir/|encryptedDirName/", list.Format(item1))
+
+}
+
+func TestDirMove(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+
+ r.Mkdir(ctx, r.Fremote)
+
+ // Make some files and dirs
+ r.ForceMkdir(ctx, r.Fremote)
+ files := []fstest.Item{
+ r.WriteObject(ctx, "A1/one", "one", t1),
+ r.WriteObject(ctx, "A1/two", "two", t2),
+ r.WriteObject(ctx, "A1/B1/three", "three", t3),
+ r.WriteObject(ctx, "A1/B1/C1/four", "four", t1),
+ r.WriteObject(ctx, "A1/B1/C2/five", "five", t2),
+ }
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2"))
+ require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C3"))
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ files,
+ []string{
+ "A1",
+ "A1/B1",
+ "A1/B2",
+ "A1/B1/C1",
+ "A1/B1/C2",
+ "A1/B1/C3",
+ },
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+
+ require.NoError(t, operations.DirMove(ctx, r.Fremote, "A1", "A2"))
+
+ for i := range files {
+ files[i].Path = strings.ReplaceAll(files[i].Path, "A1/", "A2/")
+ }
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ files,
+ []string{
+ "A2",
+ "A2/B1",
+ "A2/B2",
+ "A2/B1/C1",
+ "A2/B1/C2",
+ "A2/B1/C3",
+ },
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+
+ // Disable DirMove
+ features := r.Fremote.Features()
+ features.DirMove = nil
+
+ require.NoError(t, operations.DirMove(ctx, r.Fremote, "A2", "A3"))
+
+ for i := range files {
+ files[i].Path = strings.ReplaceAll(files[i].Path, "A2/", "A3/")
+ }
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ files,
+ []string{
+ "A3",
+ "A3/B1",
+ "A3/B2",
+ "A3/B1/C1",
+ "A3/B1/C2",
+ "A3/B1/C3",
+ },
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+
+ // Try with a DirMove method that exists but returns fs.ErrorCantDirMove (ex. combine moving across upstreams)
+ // Should fall back to manual move (copy + delete)
+
+ features.DirMove = func(ctx context.Context, src fs.Fs, srcRemote string, dstRemote string) error {
+ return fs.ErrorCantDirMove
+ }
+
+ assert.NoError(t, operations.DirMove(ctx, r.Fremote, "A3", "A4"))
+
+ for i := range files {
+ files[i].Path = strings.ReplaceAll(files[i].Path, "A3/", "A4/")
+ }
+
+ fstest.CheckListingWithPrecision(
+ t,
+ r.Fremote,
+ files,
+ []string{
+ "A4",
+ "A4/B1",
+ "A4/B2",
+ "A4/B1/C1",
+ "A4/B1/C2",
+ "A4/B1/C3",
+ },
+ fs.GetModifyWindow(ctx, r.Fremote),
+ )
+}
+
+func TestGetFsInfo(t *testing.T) {
+ r := fstest.NewRun(t)
+
+ f := r.Fremote
+ info := operations.GetFsInfo(f)
+ assert.Equal(t, f.Name(), info.Name)
+ assert.Equal(t, f.Root(), info.Root)
+ assert.Equal(t, f.String(), info.String)
+ assert.Equal(t, f.Precision(), info.Precision)
+ hashSet := hash.NewHashSet()
+ for _, hashName := range info.Hashes {
+ var ht hash.Type
+ require.NoError(t, ht.Set(hashName))
+ hashSet.Add(ht)
+ }
+ assert.Equal(t, f.Hashes(), hashSet)
+ assert.Equal(t, f.Features().Enabled(), info.Features)
+}
+
+func TestRcat(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ check := func(t *testing.T, withChecksum, ignoreChecksum bool) {
+ ci.CheckSum, ci.IgnoreChecksum = withChecksum, ignoreChecksum
+
+ var prefix string
+ if withChecksum {
+ prefix = "with_checksum_"
+ } else {
+ prefix = "no_checksum_"
+ }
+ if ignoreChecksum {
+ prefix = "ignore_checksum_"
+ }
+
+ r := fstest.NewRun(t)
+
+ if *fstest.SizeLimit > 0 && int64(ci.StreamingUploadCutoff) > *fstest.SizeLimit {
+ savedCutoff := ci.StreamingUploadCutoff
+ ci.StreamingUploadCutoff = fs.SizeSuffix(*fstest.SizeLimit)
+ t.Logf("Adjust StreamingUploadCutoff to size limit %s (was %s)", ci.StreamingUploadCutoff, savedCutoff)
+ }
+
+ fstest.CheckListing(t, r.Fremote, []fstest.Item{})
+
+ data1 := "this is some really nice test data"
+ path1 := prefix + "small_file_from_pipe"
+
+ data2 := string(make([]byte, ci.StreamingUploadCutoff+1))
+ path2 := prefix + "big_file_from_pipe"
+
+ in := io.NopCloser(strings.NewReader(data1))
+ _, err := operations.Rcat(ctx, r.Fremote, path1, in, t1, nil)
+ require.NoError(t, err)
+
+ in = io.NopCloser(strings.NewReader(data2))
+ _, err = operations.Rcat(ctx, r.Fremote, path2, in, t2, nil)
+ require.NoError(t, err)
+
+ file1 := fstest.NewItem(path1, data1, t1)
+ file2 := fstest.NewItem(path2, data2, t2)
+ r.CheckRemoteItems(t, file1, file2)
+ }
+
+ for i := range 4 {
+ withChecksum := (i & 1) != 0
+ ignoreChecksum := (i & 2) != 0
+ t.Run(fmt.Sprintf("withChecksum=%v,ignoreChecksum=%v", withChecksum, ignoreChecksum), func(t *testing.T) {
+ check(t, withChecksum, ignoreChecksum)
+ })
+ }
+}
+
+func TestRcatMetadata(t *testing.T) {
+ r := fstest.NewRun(t)
+
+ if !r.Fremote.Features().UserMetadata {
+ t.Skip("Skipping as destination doesn't support user metadata")
+ }
+
+ test := func(disableUploadCutoff bool) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ ci.Metadata = true
+ data := "this is some really nice test data with metadata"
+ path := "rcat_metadata"
+
+ meta := fs.Metadata{
+ "key": "value",
+ "sausage": "potato",
+ }
+
+ if disableUploadCutoff {
+ ci.StreamingUploadCutoff = 0
+ data += " uploadCutoff=0"
+ path += "_uploadcutoff0"
+ }
+
+ fstest.CheckListing(t, r.Fremote, []fstest.Item{})
+
+ in := io.NopCloser(strings.NewReader(data))
+ _, err := operations.Rcat(ctx, r.Fremote, path, in, t1, meta)
+ require.NoError(t, err)
+
+ file := fstest.NewItem(path, data, t1)
+ r.CheckRemoteItems(t, file)
+
+ o, err := r.Fremote.NewObject(ctx, path)
+ require.NoError(t, err)
+ gotMeta, err := fs.GetMetadata(ctx, o)
+ require.NoError(t, err)
+ // Check the specific user data we set is set
+ // Likely there will be other values
+ assert.Equal(t, "value", gotMeta["key"])
+ assert.Equal(t, "potato", gotMeta["sausage"])
+
+ // Delete the test file
+ require.NoError(t, o.Remove(ctx))
+ }
+
+ t.Run("Normal", func(t *testing.T) {
+ test(false)
+ })
+ t.Run("ViaDisk", func(t *testing.T) {
+ test(true)
+ })
+}
+
+func TestRcatSize(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+
+ const body = "------------------------------------------------------------"
+ file1 := r.WriteFile("potato1", body, t1)
+ file2 := r.WriteFile("potato2", body, t2)
+ // Test with known length
+ bodyReader := io.NopCloser(strings.NewReader(body))
+ obj, err := operations.RcatSize(ctx, r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime, nil)
+ require.NoError(t, err)
+ assert.Equal(t, int64(len(body)), obj.Size())
+ assert.Equal(t, file1.Path, obj.Remote())
+
+ // Test with unknown length
+ bodyReader = io.NopCloser(strings.NewReader(body)) // reset Reader
+ io.NopCloser(strings.NewReader(body))
+ obj, err = operations.RcatSize(ctx, r.Fremote, file2.Path, bodyReader, -1, file2.ModTime, nil)
+ require.NoError(t, err)
+ assert.Equal(t, int64(len(body)), obj.Size())
+ assert.Equal(t, file2.Path, obj.Remote())
+
+ // Check files exist
+ r.CheckRemoteItems(t, file1, file2)
+}
+
+func TestRcatSizeMetadata(t *testing.T) {
+ r := fstest.NewRun(t)
+
+ if !r.Fremote.Features().UserMetadata {
+ t.Skip("Skipping as destination doesn't support user metadata")
+ }
+
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ ci.Metadata = true
+
+ meta := fs.Metadata{
+ "key": "value",
+ "sausage": "potato",
+ }
+
+ const body = "------------------------------------------------------------"
+ file1 := r.WriteFile("potato1", body, t1)
+ file2 := r.WriteFile("potato2", body, t2)
+
+ // Test with known length
+ bodyReader := io.NopCloser(strings.NewReader(body))
+ obj, err := operations.RcatSize(ctx, r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime, meta)
+ require.NoError(t, err)
+ assert.Equal(t, int64(len(body)), obj.Size())
+ assert.Equal(t, file1.Path, obj.Remote())
+
+ // Test with unknown length
+ bodyReader = io.NopCloser(strings.NewReader(body)) // reset Reader
+ io.NopCloser(strings.NewReader(body))
+ obj, err = operations.RcatSize(ctx, r.Fremote, file2.Path, bodyReader, -1, file2.ModTime, meta)
+ require.NoError(t, err)
+ assert.Equal(t, int64(len(body)), obj.Size())
+ assert.Equal(t, file2.Path, obj.Remote())
+
+ // Check files exist
+ r.CheckRemoteItems(t, file1, file2)
+
+ // Check metadata OK
+ for _, path := range []string{file1.Path, file2.Path} {
+ o, err := r.Fremote.NewObject(ctx, path)
+ require.NoError(t, err)
+ gotMeta, err := fs.GetMetadata(ctx, o)
+ require.NoError(t, err)
+ // Check the specific user data we set is set
+ // Likely there will be other values
+ assert.Equal(t, "value", gotMeta["key"])
+ assert.Equal(t, "potato", gotMeta["sausage"])
+ }
+}
+
+func TestTouchDir(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+
+ if r.Fremote.Precision() == fs.ModTimeNotSupported {
+ t.Skip("Skipping test as remote does not support modtime")
+ }
+
+ file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
+ file2 := r.WriteBoth(ctx, "empty space", "-", t2)
+ file3 := r.WriteBoth(ctx, "sub dir/potato3", "hello", t2)
+ r.CheckRemoteItems(t, file1, file2, file3)
+
+ accounting.GlobalStats().ResetCounters()
+ timeValue := time.Date(2010, 9, 8, 7, 6, 5, 4, time.UTC)
+ err := operations.TouchDir(ctx, r.Fremote, "", timeValue, true)
+ require.NoError(t, err)
+ if accounting.Stats(ctx).GetErrors() != 0 {
+ err = accounting.Stats(ctx).GetLastError()
+ require.True(t, errors.Is(err, fs.ErrorCantSetModTime) || errors.Is(err, fs.ErrorCantSetModTimeWithoutDelete))
+ } else {
+ file1.ModTime = timeValue
+ file2.ModTime = timeValue
+ file3.ModTime = timeValue
+ r.CheckRemoteItems(t, file1, file2, file3)
+ }
+}
+
+var testMetadata = fs.Metadata{
+ // System metadata supported by all backends
+ "mtime": t1.Format(time.RFC3339Nano),
+ // User metadata
+ "potato": "jersey",
+}
+
+func TestMkdirMetadata(t *testing.T) {
+ const name = "dir with metadata"
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ ci.Metadata = true
+ r := fstest.NewRun(t)
+ features := r.Fremote.Features()
+ if features.MkdirMetadata == nil {
+ t.Skip("Skipping test as remote does not support MkdirMetadata")
+ }
+
+ newDst, err := operations.MkdirMetadata(ctx, r.Fremote, name, testMetadata)
+ require.NoError(t, err)
+ require.NotNil(t, newDst)
+
+ require.True(t, features.ReadDirMetadata, "Expecting ReadDirMetadata to be supported if MkdirMetadata is supported")
+
+ // Check the returned directory and one read from the listing
+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)
+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), testMetadata)
+}
+
+func TestMkdirModTime(t *testing.T) {
+ const name = "directory with modtime"
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil {
+ t.Skip("Skipping test as remote does not support DirSetModTime or MkdirMetadata")
+ }
+ newDst, err := operations.MkdirModTime(ctx, r.Fremote, name, t2)
+ require.NoError(t, err)
+
+ // Check the returned directory and one read from the listing
+ // newDst may be nil here depending on how the modtime was set
+ if newDst != nil {
+ fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2)
+ }
+ fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2)
+}
+
+func TestCopyDirMetadata(t *testing.T) {
+ const nameNonExistent = "non existent directory"
+ const nameExistent = "existing directory"
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ ci.Metadata = true
+ r := fstest.NewRun(t)
+ if !r.Fremote.Features().WriteDirMetadata && r.Fremote.Features().MkdirMetadata == nil {
+ t.Skip("Skipping test as remote does not support WriteDirMetadata or MkdirMetadata")
+ }
+
+ // Create a source local directory with metadata
+ newSrc, err := operations.MkdirMetadata(ctx, r.Flocal, "dir with metadata to be copied", testMetadata)
+ require.NoError(t, err)
+ require.NotNil(t, newSrc)
+
+ // First try with the directory not existing
+ newDst, err := operations.CopyDirMetadata(ctx, r.Fremote, nil, nameNonExistent, newSrc)
+ require.NoError(t, err)
+ require.NotNil(t, newDst)
+
+ // Check the returned directory and one read from the listing
+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)
+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameNonExistent), testMetadata)
+
+ // Then try with the directory existing
+ require.NoError(t, r.Fremote.Rmdir(ctx, nameNonExistent))
+ require.NoError(t, r.Fremote.Mkdir(ctx, nameExistent))
+ existingDir := fstest.NewDirectory(ctx, t, r.Fremote, nameExistent)
+
+ newDst, err = operations.CopyDirMetadata(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", newSrc)
+ require.NoError(t, err)
+ require.NotNil(t, newDst)
+
+ // Check the returned directory and one read from the listing
+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata)
+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameExistent), testMetadata)
+}
+
+func TestSetDirModTime(t *testing.T) {
+ const name = "set modtime on existing directory"
+ ctx, ci := fs.AddConfig(context.Background())
+ r := fstest.NewRun(t)
+ if r.Fremote.Features().DirSetModTime == nil && !r.Fremote.Features().WriteDirSetModTime {
+ t.Skip("Skipping test as remote does not support DirSetModTime or WriteDirSetModTime")
+ }
+
+ // Check that we obey --no-update-dir-modtime - this should return nil, nil
+ ci.NoUpdateDirModTime = true
+ newDst, err := operations.SetDirModTime(ctx, r.Fremote, nil, "set modtime on non existent directory", t2)
+ require.NoError(t, err)
+ require.Nil(t, newDst)
+ ci.NoUpdateDirModTime = false
+
+ // First try with the directory not existing - should return an error
+ newDst, err = operations.SetDirModTime(ctx, r.Fremote, nil, "set modtime on non existent directory", t2)
+ require.Error(t, err)
+ require.Nil(t, newDst)
+
+ // Then try with the directory existing
+ require.NoError(t, r.Fremote.Mkdir(ctx, name))
+ existingDir := fstest.NewDirectory(ctx, t, r.Fremote, name)
+
+ newDst, err = operations.SetDirModTime(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", t2)
+ require.NoError(t, err)
+ require.NotNil(t, newDst)
+
+ // Check the returned directory and one read from the listing
+ // The modtime will only be correct on newDst if it had a SetModTime method
+ if _, ok := newDst.(fs.SetModTimer); ok {
+ fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2)
+ }
+ fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2)
+
+ // Now wrap the directory to make the SetModTime method return fs.ErrorNotImplemented and check that it falls back correctly
+ wrappedDir := fs.NewDirWrapper(existingDir.Remote(), fs.NewDir(existingDir.Remote(), existingDir.ModTime(ctx)))
+ newDst, err = operations.SetDirModTime(ctx, r.Fremote, wrappedDir, "SHOULD BE IGNORED", t1)
+ require.NoError(t, err)
+ require.NotNil(t, newDst)
+ fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t1)
+}
+
+func TestDirsEqual(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ ci.Metadata = true
+ r := fstest.NewRun(t)
+ if !r.Fremote.Features().WriteDirMetadata && r.Fremote.Features().MkdirMetadata == nil {
+ t.Skip("Skipping test as remote does not support WriteDirMetadata or MkdirMetadata")
+ }
+
+ opt := operations.DirsEqualOpt{
+ ModifyWindow: fs.GetModifyWindow(ctx, r.Flocal, r.Fremote),
+ SetDirModtime: true,
+ SetDirMetadata: true,
+ }
+
+ // Create a source local directory with metadata
+ src, err := operations.MkdirMetadata(ctx, r.Flocal, "dir with metadata to be copied", testMetadata)
+ require.NoError(t, err)
+ require.NotNil(t, src)
+
+ // try with nil dst -- should be false
+ equal := operations.DirsEqual(ctx, src, nil, opt)
+ assert.False(t, equal)
+
+ // make a dest with an equal modtime
+ dst, err := operations.MkdirModTime(ctx, r.Fremote, "dst", src.ModTime(ctx))
+ require.NoError(t, err)
+
+ // try with equal modtimes -- should be true
+ equal = operations.DirsEqual(ctx, src, dst, opt)
+ assert.True(t, equal)
+
+ // try with unequal modtimes -- should be false
+ dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", t2)
+ require.NoError(t, err)
+ equal = operations.DirsEqual(ctx, src, dst, opt)
+ assert.False(t, equal)
+
+ // try with unequal modtimes that are within modify window -- should be true
+ halfWindow := opt.ModifyWindow / 2
+ dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", src.ModTime(ctx).Add(halfWindow))
+ require.NoError(t, err)
+ equal = operations.DirsEqual(ctx, src, dst, opt)
+ assert.True(t, equal)
+
+ // test ignoretimes -- should be false
+ ci.IgnoreTimes = true
+ equal = operations.DirsEqual(ctx, src, dst, opt)
+ assert.False(t, equal)
+
+ // test immutable -- should be true
+ ci.IgnoreTimes = false
+ ci.Immutable = true
+ dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", t3)
+ require.NoError(t, err)
+ equal = operations.DirsEqual(ctx, src, dst, opt)
+ assert.True(t, equal)
+
+ // test dst newer than src with --update -- should be true
+ ci.Immutable = false
+ ci.UpdateOlder = true
+ equal = operations.DirsEqual(ctx, src, dst, opt)
+ assert.True(t, equal)
+
+ // test no SetDirModtime or SetDirMetadata -- should be true
+ ci.UpdateOlder = false
+ opt.SetDirMetadata, opt.SetDirModtime = false, false
+ equal = operations.DirsEqual(ctx, src, dst, opt)
+ assert.True(t, equal)
+}
+
+func TestRemoveExisting(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ if r.Fremote.Features().Move == nil {
+ t.Skip("Skipping as remote can't Move")
+ }
+
+ file1 := r.WriteObject(ctx, "sub dir/test remove existing", "hello world", t1)
+ file2 := r.WriteObject(ctx, "sub dir/test remove existing with long name 123456789012345678901234567890123456789012345678901234567890123456789", "hello long name world", t1)
+
+ r.CheckRemoteItems(t, file1, file2)
+
+ var returnedError error
+
+ // Check not found first
+ cleanup, err := operations.RemoveExisting(ctx, r.Fremote, "not found", "TEST")
+ assert.Equal(t, err, nil)
+ r.CheckRemoteItems(t, file1, file2)
+ cleanup(&returnedError)
+ r.CheckRemoteItems(t, file1, file2)
+
+ // Remove file1
+ cleanup, err = operations.RemoveExisting(ctx, r.Fremote, file1.Path, "TEST")
+ assert.Equal(t, err, nil)
+ //r.CheckRemoteItems(t, file1, file2)
+
+ // Check file1 with temporary name exists
+ var buf bytes.Buffer
+ err = operations.List(ctx, r.Fremote, &buf)
+ require.NoError(t, err)
+ res := buf.String()
+ assert.NotContains(t, res, " 11 "+file1.Path+"\n")
+ assert.Contains(t, res, " 11 "+file1.Path+".")
+ assert.Contains(t, res, " 21 "+file2.Path+"\n")
+
+ cleanup(&returnedError)
+ r.CheckRemoteItems(t, file2)
+
+ // Remove file2 with an error
+ cleanup, err = operations.RemoveExisting(ctx, r.Fremote, file2.Path, "TEST")
+ assert.Equal(t, err, nil)
+
+ // Check file2 with truncated temporary name exists
+ buf.Reset()
+ err = operations.List(ctx, r.Fremote, &buf)
+ require.NoError(t, err)
+ res = buf.String()
+ assert.NotContains(t, res, " 21 "+file2.Path+"\n")
+ assert.NotContains(t, res, " 21 "+file2.Path+".")
+ assert.Contains(t, res, " 21 "+file2.Path[:100])
+
+ returnedError = errors.New("BOOM")
+ cleanup(&returnedError)
+ r.CheckRemoteItems(t, file2)
+
+ // Remove file2
+ cleanup, err = operations.RemoveExisting(ctx, r.Fremote, file2.Path, "TEST")
+ assert.Equal(t, err, nil)
+
+ // Check file2 with truncated temporary name exists
+ buf.Reset()
+ err = operations.List(ctx, r.Fremote, &buf)
+ require.NoError(t, err)
+ res = buf.String()
+ assert.NotContains(t, res, " 21 "+file2.Path+"\n")
+ assert.NotContains(t, res, " 21 "+file2.Path+".")
+ assert.Contains(t, res, " 21 "+file2.Path[:100])
+
+ returnedError = nil
+ cleanup(&returnedError)
+ r.CheckRemoteItems(t)
+}
diff --git a/fs/operations/operationsflags/operationsflags.go b/fs/operations/operationsflags/operationsflags.go
new file mode 100644
index 0000000..054ed96
--- /dev/null
+++ b/fs/operations/operationsflags/operationsflags.go
@@ -0,0 +1,147 @@
+// Package operationsflags defines the flags used by rclone operations.
+// It is decoupled into a separate package so it can be replaced.
+package operationsflags
+
+import (
+ "context"
+ _ "embed"
+ "io"
+ "os"
+ "strings"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/config/flags"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/spf13/cobra"
+ "github.com/spf13/pflag"
+)
+
+//go:embed operationsflags.md
+var help string
+
+// Help returns the help string cleaned up to simplify appending
+func Help() string {
+ return strings.TrimSpace(help)
+}
+
+// AddLoggerFlagsOptions contains options for the Logger Flags
+type AddLoggerFlagsOptions struct {
+ Combined string // a file with file names with leading sigils
+ MissingOnSrc string // files only in the destination
+ MissingOnDst string // files only in the source
+ Match string // matching files
+ Differ string // differing files
+ ErrFile string // files with errors of some kind
+ DestAfter string // files that exist on the destination post-sync
+}
+
+// AnySet checks if any of the logger flags have a non-blank value
+func (o AddLoggerFlagsOptions) AnySet() bool {
+ return anyNotBlank(o.Combined, o.MissingOnSrc, o.MissingOnDst, o.Match, o.Differ, o.ErrFile, o.DestAfter)
+}
+
+func anyNotBlank(s ...string) bool {
+ for _, x := range s {
+ if x != "" {
+ return true
+ }
+ }
+ return false
+}
+
+// AddLoggerFlags adds the logger flags to the cmdFlags command
+func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt *AddLoggerFlagsOptions) {
+ flags.StringVarP(cmdFlags, &flagsOpt.Combined, "combined", "", flagsOpt.Combined, "Make a combined report of changes to this file", "Sync")
+ flags.StringVarP(cmdFlags, &flagsOpt.MissingOnSrc, "missing-on-src", "", flagsOpt.MissingOnSrc, "Report all files missing from the source to this file", "Sync")
+ flags.StringVarP(cmdFlags, &flagsOpt.MissingOnDst, "missing-on-dst", "", flagsOpt.MissingOnDst, "Report all files missing from the destination to this file", "Sync")
+ flags.StringVarP(cmdFlags, &flagsOpt.Match, "match", "", flagsOpt.Match, "Report all matching files to this file", "Sync")
+ flags.StringVarP(cmdFlags, &flagsOpt.Differ, "differ", "", flagsOpt.Differ, "Report all non-matching files to this file", "Sync")
+ flags.StringVarP(cmdFlags, &flagsOpt.ErrFile, "error", "", flagsOpt.ErrFile, "Report all files with errors (hashing or reading) to this file", "Sync")
+ flags.StringVarP(cmdFlags, &flagsOpt.DestAfter, "dest-after", "", flagsOpt.DestAfter, "Report all files that exist on the dest post-sync", "Sync")
+
+ // lsf flags for destAfter
+ flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync")
+ flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", "", "Specify a custom time format - see docs for details (default: 2006-01-02 15:04:05)", "")
+ flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync")
+ opt.HashType = hash.MD5
+ flags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync")
+ flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync")
+ // flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "")
+}
+
+// ConfigureLoggers verifies and sets up writers for log files requested via CLI flags
+func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, opt *operations.LoggerOpt, flagsOpt AddLoggerFlagsOptions) (func(), error) {
+ closers := []io.Closer{}
+
+ if opt.TimeFormat == "max" {
+ opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())
+ }
+ opt.SetListFormat(ctx, command.Flags())
+ opt.NewListJSON(ctx, fdst, "")
+
+ open := func(name string, pout *io.Writer) error {
+ if name == "" {
+ return nil
+ }
+ if name == "-" {
+ *pout = os.Stdout
+ return nil
+ }
+ out, err := os.Create(name)
+ if err != nil {
+ return err
+ }
+ *pout = out
+ closers = append(closers, out)
+ return nil
+ }
+
+ if err := open(flagsOpt.Combined, &opt.Combined); err != nil {
+ return nil, err
+ }
+ if err := open(flagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil {
+ return nil, err
+ }
+ if err := open(flagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil {
+ return nil, err
+ }
+ if err := open(flagsOpt.Match, &opt.Match); err != nil {
+ return nil, err
+ }
+ if err := open(flagsOpt.Differ, &opt.Differ); err != nil {
+ return nil, err
+ }
+ if err := open(flagsOpt.ErrFile, &opt.Error); err != nil {
+ return nil, err
+ }
+ if err := open(flagsOpt.DestAfter, &opt.DestAfter); err != nil {
+ return nil, err
+ }
+
+ close := func() {
+ for _, closer := range closers {
+ err := closer.Close()
+ if err != nil {
+ fs.Errorf(nil, "Failed to close report output: %v", err)
+ }
+ }
+ }
+
+ ci := fs.GetConfig(ctx)
+ if ci.NoTraverse && opt.Combined != nil {
+ fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n")
+ }
+ if ci.NoTraverse && opt.MissingOnSrc != nil {
+ fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n")
+ }
+ if ci.NoTraverse && opt.DestAfter != nil {
+ fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n")
+ }
+
+ return close, nil
+}
diff --git a/fs/operations/operationsflags/operationsflags.md b/fs/operations/operationsflags/operationsflags.md
new file mode 100644
index 0000000..43def48
--- /dev/null
+++ b/fs/operations/operationsflags/operationsflags.md
@@ -0,0 +1,40 @@
+### Logger Flags
+
+The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match` and `--error`
+flags write paths, one per line, to the file name (or stdout if it is `-`)
+supplied. What they write is described in the help below. For example
+`--differ` will write all paths which are present on both the source and
+destination but different.
+
+The `--combined` flag will write a file (or stdout) which contains all
+file paths with a symbol and then a space and then the path to tell
+you what happened to it. These are reminiscent of diff files.
+
+- `= path` means path was found in source and destination and was identical
+- `- path` means path was missing on the source, so only in the destination
+- `+ path` means path was missing on the destination, so only in the source
+- `* path` means path was present in source and destination but different.
+- `! path` means there was an error reading or hashing the source or dest.
+
+The `--dest-after` flag writes a list file using the same format flags
+as [`lsf`](/commands/rclone_lsf/#synopsis) (including [customizable options
+for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
+Conceptually it is similar to rsync's `--itemize-changes`, but not identical
+-- it should output an accurate list of what will be on the destination
+after the command is finished.
+
+When the `--no-traverse` flag is set, all logs involving files that exist only
+on the destination will be incomplete or completely missing.
+
+Note that these logger flags have a few limitations, and certain scenarios
+are not currently supported:
+
+- `--max-duration` / `CutoffModeHard`
+- `--compare-dest` / `--copy-dest`
+- server-side moves of an entire dir at once
+- High-level retries, because there would be duplicates (use `--retries 1` to disable)
+- Possibly some unusual error scenarios
+
+Note also that each file is logged during execution, as opposed to after, so it
+is most useful as a predictor of what SHOULD happen to each file
+(which may or may not match what actually DID).
diff --git a/fs/operations/rc.go b/fs/operations/rc.go
new file mode 100644
index 0000000..ce92a00
--- /dev/null
+++ b/fs/operations/rc.go
@@ -0,0 +1,1016 @@
+package operations
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "mime"
+ "mime/multipart"
+ "net/http"
+ "path"
+ "strings"
+ "time"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/config"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/rc"
+ "github.com/rclone/rclone/lib/diskusage"
+)
+
+func init() {
+ rc.Add(rc.Call{
+ Path: "operations/list",
+ AuthRequired: true,
+ Fn: rcList,
+ Title: "List the given remote and path in JSON format",
+ Help: `This takes the following parameters:
+
+- fs - a remote name string e.g. "drive:"
+- remote - a path within that remote e.g. "dir"
+- opt - a dictionary of options to control the listing (optional)
+ - recurse - If set recurse directories
+ - noModTime - If set return modification time
+ - showEncrypted - If set show decrypted names
+ - showOrigIDs - If set show the IDs for each item if known
+ - showHash - If set return a dictionary of hashes
+ - noMimeType - If set don't show mime types
+ - dirsOnly - If set only show directories
+ - filesOnly - If set only show files
+ - metadata - If set return metadata of objects also
+ - hashTypes - array of strings of hash types to show if showHash set
+
+Returns:
+
+- list
+ - This is an array of objects as described in the lsjson command
+
+See the [lsjson](/commands/rclone_lsjson/) command for more information on the above and examples.
+`,
+ })
+}
+
+// List the directory
+func rcList(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+ f, remote, err := rc.GetFsAndRemote(ctx, in)
+ if err != nil {
+ return nil, err
+ }
+ var opt ListJSONOpt
+ err = in.GetStruct("opt", &opt)
+ if rc.NotErrParamNotFound(err) {
+ return nil, err
+ }
+ list := []*ListJSONItem{}
+ err = ListJSON(ctx, f, remote, &opt, func(item *ListJSONItem) error {
+ list = append(list, item)
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ out = make(rc.Params)
+ out["list"] = list
+ return out, nil
+}
+
+func init() {
+ rc.Add(rc.Call{
+ Path: "operations/stat",
+ AuthRequired: true,
+ Fn: rcStat,
+ Title: "Give information about the supplied file or directory",
+ Help: `This takes the following parameters
+
+- fs - a remote name string eg "drive:"
+- remote - a path within that remote eg "dir"
+- opt - a dictionary of options to control the listing (optional)
+ - see operations/list for the options
+
+The result is
+
+- item - an object as described in the lsjson command. Will be null if not found.
+
+Note that if you are only interested in files then it is much more
+efficient to set the filesOnly flag in the options.
+
+See the [lsjson](/commands/rclone_lsjson/) command for more information on the above and examples.
+`,
+ })
+}
+
+// List the directory
+func rcStat(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+ f, remote, err := rc.GetFsAndRemote(ctx, in)
+ if err != nil {
+ return nil, err
+ }
+ var opt ListJSONOpt
+ err = in.GetStruct("opt", &opt)
+ if rc.NotErrParamNotFound(err) {
+ return nil, err
+ }
+ item, err := StatJSON(ctx, f, remote, &opt)
+ if err != nil {
+ return nil, err
+ }
+ out = make(rc.Params)
+ out["item"] = item
+ return out, nil
+}
+
+func init() {
+ rc.Add(rc.Call{
+ Path: "operations/about",
+ AuthRequired: true,
+ Fn: rcAbout,
+ Title: "Return the space used on the remote",
+ Help: `This takes the following parameters:
+
+- fs - a remote name string e.g. "drive:"
+
+The result is as returned from rclone about --json
+
+See the [about](/commands/rclone_about/) command for more information on the above.
+`,
+ })
+}
+
+// About the remote
+func rcAbout(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+ f, err := rc.GetFs(ctx, in)
+ if err != nil {
+ return nil, err
+ }
+ doAbout := f.Features().About
+ if doAbout == nil {
+ return nil, fmt.Errorf("%v doesn't support about", f)
+ }
+ u, err := doAbout(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("about call failed: %w", err)
+ }
+ err = rc.Reshape(&out, u)
+ if err != nil {
+ return nil, fmt.Errorf("about Reshape failed: %w", err)
+ }
+ return out, nil
+}
+
+func init() {
+ for _, copy := range []bool{false, true} {
+ name := "Move"
+ if copy {
+ name = "Copy"
+ }
+ rc.Add(rc.Call{
+ Path: "operations/" + strings.ToLower(name) + "file",
+ AuthRequired: true,
+ Fn: func(ctx context.Context, in rc.Params) (rc.Params, error) {
+ return rcMoveOrCopyFile(ctx, in, copy)
+ },
+ Title: name + " a file from source remote to destination remote",
+ Help: `This takes the following parameters:
+
+- srcFs - a remote name string e.g. "drive:" for the source, "/" for local filesystem
+- srcRemote - a path within that remote e.g. "file.txt" for the source
+- dstFs - a remote name string e.g. "drive2:" for the destination, "/" for local filesystem
+- dstRemote - a path within that remote e.g. "file2.txt" for the destination
+`,
+ })
+ }
+}
+
+// Copy a file
+func rcMoveOrCopyFile(ctx context.Context, in rc.Params, cp bool) (out rc.Params, err error) {
+ srcFs, srcRemote, err := rc.GetFsAndRemoteNamed(ctx, in, "srcFs", "srcRemote")
+ if err != nil {
+ return nil, err
+ }
+ dstFs, dstRemote, err := rc.GetFsAndRemoteNamed(ctx, in, "dstFs", "dstRemote")
+ if err != nil {
+ return nil, err
+ }
+ return nil, moveOrCopyFile(ctx, dstFs, srcFs, dstRemote, srcRemote, cp, false)
+}
+
+func init() {
+ for _, op := range []struct {
+ name string
+ title string
+ help string
+ noRemote bool
+ needsRequest bool
+ noCommand bool
+ }{
+ {name: "mkdir", title: "Make a destination directory or container"},
+ {name: "rmdir", title: "Remove an empty directory or container"},
+ {name: "purge", title: "Remove a directory or container and all of its contents"},
+ {name: "rmdirs", title: "Remove all the empty directories in the path", help: "- leaveRoot - boolean, set to true not to delete the root\n"},
+ {name: "delete", title: "Remove files in the path", noRemote: true},
+ {name: "deletefile", title: "Remove the single file pointed to"},
+ {name: "copyurl", title: "Copy the URL to the object", help: "- url - string, URL to read from\n - autoFilename - boolean, set to true to retrieve destination file name from url\n"},
+ {name: "uploadfile", title: "Upload file using multiform/form-data", help: "- each part in body represents a file to be uploaded\n", needsRequest: true, noCommand: true},
+ {name: "cleanup", title: "Remove trashed files in the remote or path", noRemote: true},
+ {name: "settier", title: "Changes storage tier or class on all files in the path", noRemote: true},
+ {name: "settierfile", title: "Changes storage tier or class on the single file pointed to", noCommand: true},
+ } {
+ var remote, command string
+ if !op.noRemote {
+ remote = "- remote - a path within that remote e.g. \"dir\"\n"
+ }
+ if !op.noCommand {
+ command = "See the [" + op.name + "](/commands/rclone_" + op.name + "/) command for more information on the above.\n"
+ }
+ rc.Add(rc.Call{
+ Path: "operations/" + op.name,
+ AuthRequired: true,
+ NeedsRequest: op.needsRequest,
+ Fn: func(ctx context.Context, in rc.Params) (rc.Params, error) {
+ return rcSingleCommand(ctx, in, op.name, op.noRemote)
+ },
+ Title: op.title,
+ Help: `This takes the following parameters:
+
+- fs - a remote name string e.g. "drive:"
+` + remote + op.help + "\n" + command,
+ })
+ }
+}
+
+// Run a single command, e.g. Mkdir
+func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bool) (out rc.Params, err error) {
+ var (
+ f fs.Fs
+ remote string
+ )
+ if noRemote {
+ f, err = rc.GetFs(ctx, in)
+ } else {
+ f, remote, err = rc.GetFsAndRemote(ctx, in)
+ }
+ if err != nil {
+ return nil, err
+ }
+ switch name {
+ case "mkdir":
+ return nil, Mkdir(ctx, f, remote)
+ case "rmdir":
+ return nil, Rmdir(ctx, f, remote)
+ case "purge":
+ return nil, Purge(ctx, f, remote)
+ case "rmdirs":
+ leaveRoot, err := in.GetBool("leaveRoot")
+ if rc.NotErrParamNotFound(err) {
+ return nil, err
+ }
+ return nil, Rmdirs(ctx, f, remote, leaveRoot)
+ case "delete":
+ return nil, Delete(ctx, f)
+ case "deletefile":
+ o, err := f.NewObject(ctx, remote)
+ if err != nil {
+ return nil, err
+ }
+ return nil, DeleteFile(ctx, o)
+ case "copyurl":
+ url, err := in.GetString("url")
+ if err != nil {
+ return nil, err
+ }
+ autoFilename, _ := in.GetBool("autoFilename")
+ noClobber, _ := in.GetBool("noClobber")
+ headerFilename, _ := in.GetBool("headerFilename")
+
+ _, err = CopyURL(ctx, f, remote, url, autoFilename, headerFilename, noClobber)
+ return nil, err
+ case "uploadfile":
+
+ var request *http.Request
+ request, err := in.GetHTTPRequest()
+ if err != nil {
+ return nil, err
+ }
+
+ contentType := request.Header.Get("Content-Type")
+ mediaType, params, err := mime.ParseMediaType(contentType)
+ if err != nil {
+ return nil, err
+ }
+
+ if strings.HasPrefix(mediaType, "multipart/") {
+ mr := multipart.NewReader(request.Body, params["boundary"])
+ for {
+ p, err := mr.NextPart()
+ if err == io.EOF {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ if p.FileName() != "" {
+ obj, err := Rcat(ctx, f, path.Join(remote, p.FileName()), p, time.Now(), nil)
+ if err != nil {
+ return nil, err
+ }
+ fs.Debugf(obj, "Upload Succeeded")
+ }
+ }
+ }
+ return nil, nil
+ case "cleanup":
+ return nil, CleanUp(ctx, f)
+ case "settier":
+ if !f.Features().SetTier {
+ return nil, fmt.Errorf("remote %s does not support settier", f.Name())
+ }
+ tier, err := in.GetString("tier")
+ if err != nil {
+ return nil, err
+ }
+ return nil, SetTier(ctx, f, tier)
+ case "settierfile":
+ if !f.Features().SetTier {
+ return nil, fmt.Errorf("remote %s does not support settier", f.Name())
+ }
+ tier, err := in.GetString("tier")
+ if err != nil {
+ return nil, err
+ }
+ o, err := f.NewObject(ctx, remote)
+ if err != nil {
+ return nil, err
+ }
+ return nil, SetTierFile(ctx, o, tier)
+ }
+ panic("unknown rcSingleCommand type")
+}
+
+func init() {
+ rc.Add(rc.Call{
+ Path: "operations/size",
+ AuthRequired: true,
+ Fn: rcSize,
+ Title: "Count the number of bytes and files in remote",
+ Help: `This takes the following parameters:
+
+- fs - a remote name string e.g. "drive:path/to/dir"
+
+Returns:
+
+- count - number of files
+- bytes - number of bytes in those files
+
+See the [size](/commands/rclone_size/) command for more information on the above.
+`,
+ })
+}
+
+// Size a directory
+func rcSize(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+ f, err := rc.GetFs(ctx, in)
+ if err != nil {
+ return nil, err
+ }
+ count, bytes, sizeless, err := Count(ctx, f)
+ if err != nil {
+ return nil, err
+ }
+ out = make(rc.Params)
+ out["count"] = count
+ out["bytes"] = bytes
+ out["sizeless"] = sizeless
+ return out, nil
+}
+
+func init() {
+ rc.Add(rc.Call{
+ Path: "operations/publiclink",
+ AuthRequired: true,
+ Fn: rcPublicLink,
+ Title: "Create or retrieve a public link to the given file or folder.",
+ Help: `This takes the following parameters:
+
+- fs - a remote name string e.g. "drive:"
+- remote - a path within that remote e.g. "dir"
+- unlink - boolean - if set removes the link rather than adding it (optional)
+- expire - string - the expiry time of the link e.g. "1d" (optional)
+
+Returns:
+
+- url - URL of the resource
+
+See the [link](/commands/rclone_link/) command for more information on the above.
+`,
+ })
+}
+
+// Make a public link
+func rcPublicLink(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+ f, remote, err := rc.GetFsAndRemote(ctx, in)
+ if err != nil {
+ return nil, err
+ }
+ unlink, _ := in.GetBool("unlink")
+ expire, err := in.GetDuration("expire")
+ if rc.IsErrParamNotFound(err) {
+ expire = time.Duration(fs.DurationOff)
+ } else if err != nil {
+ return nil, err
+ }
+ url, err := PublicLink(ctx, f, remote, fs.Duration(expire), unlink)
+ if err != nil {
+ return nil, err
+ }
+ out = make(rc.Params)
+ out["url"] = url
+ return out, nil
+}
+
+func init() {
+ rc.Add(rc.Call{
+ Path: "operations/fsinfo",
+ Fn: rcFsInfo,
+ Title: "Return information about the remote",
+ Help: `This takes the following parameters:
+
+- fs - a remote name string e.g. "drive:"
+
+This returns info about the remote passed in;
+
+` + "```" + `
+{
+ // optional features and whether they are available or not
+ "Features": {
+ "About": true,
+ "BucketBased": false,
+ "BucketBasedRootOK": false,
+ "CanHaveEmptyDirectories": true,
+ "CaseInsensitive": false,
+ "ChangeNotify": false,
+ "CleanUp": false,
+ "Command": true,
+ "Copy": false,
+ "DirCacheFlush": false,
+ "DirMove": true,
+ "Disconnect": false,
+ "DuplicateFiles": false,
+ "GetTier": false,
+ "IsLocal": true,
+ "ListR": false,
+ "MergeDirs": false,
+ "MetadataInfo": true,
+ "Move": true,
+ "OpenWriterAt": true,
+ "PublicLink": false,
+ "Purge": true,
+ "PutStream": true,
+ "PutUnchecked": false,
+ "ReadMetadata": true,
+ "ReadMimeType": false,
+ "ServerSideAcrossConfigs": false,
+ "SetTier": false,
+ "SetWrapper": false,
+ "Shutdown": false,
+ "SlowHash": true,
+ "SlowModTime": false,
+ "UnWrap": false,
+ "UserInfo": false,
+ "UserMetadata": true,
+ "WrapFs": false,
+ "WriteMetadata": true,
+ "WriteMimeType": false
+ },
+ // Names of hashes available
+ "Hashes": [
+ "md5",
+ "sha1",
+ "whirlpool",
+ "crc32",
+ "sha256",
+ "dropbox",
+ "mailru",
+ "quickxor"
+ ],
+ "Name": "local", // Name as created
+ "Precision": 1, // Precision of timestamps in ns
+ "Root": "/", // Path as created
+ "String": "Local file system at /", // how the remote will appear in logs
+ // Information about the system metadata for this backend
+ "MetadataInfo": {
+ "System": {
+ "atime": {
+ "Help": "Time of last access",
+ "Type": "RFC 3339",
+ "Example": "2006-01-02T15:04:05.999999999Z07:00"
+ },
+ "btime": {
+ "Help": "Time of file birth (creation)",
+ "Type": "RFC 3339",
+ "Example": "2006-01-02T15:04:05.999999999Z07:00"
+ },
+ "gid": {
+ "Help": "Group ID of owner",
+ "Type": "decimal number",
+ "Example": "500"
+ },
+ "mode": {
+ "Help": "File type and mode",
+ "Type": "octal, unix style",
+ "Example": "0100664"
+ },
+ "mtime": {
+ "Help": "Time of last modification",
+ "Type": "RFC 3339",
+ "Example": "2006-01-02T15:04:05.999999999Z07:00"
+ },
+ "rdev": {
+ "Help": "Device ID (if special file)",
+ "Type": "hexadecimal",
+ "Example": "1abc"
+ },
+ "uid": {
+ "Help": "User ID of owner",
+ "Type": "decimal number",
+ "Example": "500"
+ }
+ },
+ "Help": "Textual help string\n"
+ }
+}
+` + "```" + `
+
+This command does not have a command line equivalent so use this instead:
+
+ rclone rc --loopback operations/fsinfo fs=remote:
+
+`,
+ })
+}
+
+// Fsinfo the remote
+func rcFsInfo(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+ f, err := rc.GetFs(ctx, in)
+ if err != nil {
+ return nil, err
+ }
+ info := GetFsInfo(f)
+ err = rc.Reshape(&out, info)
+ if err != nil {
+ return nil, fmt.Errorf("fsinfo Reshape failed: %w", err)
+ }
+ return out, nil
+}
+
+func init() {
+ rc.Add(rc.Call{
+ Path: "backend/command",
+ AuthRequired: true,
+ Fn: rcBackend,
+ Title: "Runs a backend command.",
+ Help: `This takes the following parameters:
+
+- command - a string with the command name
+- fs - a remote name string e.g. "drive:"
+- arg - a list of arguments for the backend command
+- opt - a map of string to string of options
+
+Returns:
+
+- result - result from the backend command
+
+Example:
+
+ rclone rc backend/command command=noop fs=. -o echo=yes -o blue -a path1 -a path2
+
+Returns
+
+` + "```" + `
+{
+ "result": {
+ "arg": [
+ "path1",
+ "path2"
+ ],
+ "name": "noop",
+ "opt": {
+ "blue": "",
+ "echo": "yes"
+ }
+ }
+}
+` + "```" + `
+
+Note that this is the direct equivalent of using this "backend"
+command:
+
+ rclone backend noop . -o echo=yes -o blue path1 path2
+
+Note that arguments must be preceded by the "-a" flag
+
+See the [backend](/commands/rclone_backend/) command for more information.
+`,
+ })
+}
+
+// Make a public link
+func rcBackend(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+ f, err := rc.GetFs(ctx, in)
+ if err != nil {
+ return nil, err
+ }
+ doCommand := f.Features().Command
+ if doCommand == nil {
+ return nil, fmt.Errorf("%v: doesn't support backend commands", f)
+ }
+ command, err := in.GetString("command")
+ if err != nil {
+ return nil, err
+ }
+ opt := map[string]string{}
+ err = in.GetStructMissingOK("opt", &opt)
+ if err != nil {
+ return nil, err
+ }
+ arg := []string{}
+ err = in.GetStructMissingOK("arg", &arg)
+ if err != nil {
+ return nil, err
+ }
+ result, err := doCommand(ctx, command, arg, opt)
+ if err != nil {
+ return nil, fmt.Errorf("command %q failed: %w", command, err)
+ }
+ out = make(rc.Params)
+ out["result"] = result
+ return out, nil
+}
+
+// This should really be in fs/rc/internal.go but can't go there due
+// to a circular dependency on config.
+func init() {
+ rc.Add(rc.Call{
+ Path: "core/du",
+ Fn: rcDu,
+ Title: "Returns disk usage of a locally attached disk.",
+ Help: `
+This returns the disk usage for the local directory passed in as dir.
+
+If the directory is not passed in, it defaults to the directory
+pointed to by --cache-dir.
+
+- dir - string (optional)
+
+Returns:
+
+` + "```" + `
+{
+ "dir": "/",
+ "info": {
+ "Available": 361769115648,
+ "Free": 361785892864,
+ "Total": 982141468672
+ }
+}
+` + "```" + `
+`,
+ })
+}
+
+// Terminates app
+func rcDu(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+ dir, err := in.GetString("dir")
+ if rc.IsErrParamNotFound(err) {
+ dir = config.GetCacheDir()
+ } else if err != nil {
+ return nil, err
+ }
+ info, err := diskusage.New(dir)
+ if err != nil {
+ return nil, err
+ }
+ out = rc.Params{
+ "dir": dir,
+ "info": info,
+ }
+ return out, nil
+}
+
+func init() {
+ rc.Add(rc.Call{
+ Path: "operations/check",
+ AuthRequired: true,
+ Fn: rcCheck,
+ Title: "check the source and destination are the same",
+ Help: `Checks the files in the source and destination match. It compares
+sizes and hashes and logs a report of files that don't
+match. It doesn't alter the source or destination.
+
+This takes the following parameters:
+
+- srcFs - a remote name string e.g. "drive:" for the source, "/" for local filesystem
+- dstFs - a remote name string e.g. "drive2:" for the destination, "/" for local filesystem
+- download - check by downloading rather than with hash
+- checkFileHash - treat checkFileFs:checkFileRemote as a SUM file with hashes of given type
+- checkFileFs - treat checkFileFs:checkFileRemote as a SUM file with hashes of given type
+- checkFileRemote - treat checkFileFs:checkFileRemote as a SUM file with hashes of given type
+- oneWay - check one way only, source files must exist on remote
+- combined - make a combined report of changes (default false)
+- missingOnSrc - report all files missing from the source (default true)
+- missingOnDst - report all files missing from the destination (default true)
+- match - report all matching files (default false)
+- differ - report all non-matching files (default true)
+- error - report all files with errors (hashing or reading) (default true)
+
+If you supply the download flag, it will download the data from
+both remotes and check them against each other on the fly. This can
+be useful for remotes that don't support hashes or if you really want
+to check all the data.
+
+If you supply the size-only global flag, it will only compare the sizes not
+the hashes as well. Use this for a quick check.
+
+If you supply the checkFileHash option with a valid hash name, the
+checkFileFs:checkFileRemote must point to a text file in the SUM
+format. This treats the checksum file as the source and dstFs as the
+destination. Note that srcFs is not used and should not be supplied in
+this case.
+
+Returns:
+
+- success - true if no error, false otherwise
+- status - textual summary of check, OK or text string
+- hashType - hash used in check, may be missing
+- combined - array of strings of combined report of changes
+- missingOnSrc - array of strings of all files missing from the source
+- missingOnDst - array of strings of all files missing from the destination
+- match - array of strings of all matching files
+- differ - array of strings of all non-matching files
+- error - array of strings of all files with errors (hashing or reading)
+
+`,
+ })
+}
+
+// Writer which writes into the slice provided
+type stringWriter struct {
+ out *[]string
+}
+
+// Write writes len(p) bytes from p to the underlying data stream. It returns
+// the number of bytes written from p (0 <= n <= len(p)) and any error
+// encountered that caused the write to stop early. Write must return a non-nil
+// error if it returns n < len(p). Write must not modify the slice data,
+// even temporarily.
+//
+// Implementations must not retain p.
+func (s stringWriter) Write(p []byte) (n int, err error) {
+ result := string(p)
+ result = strings.TrimSuffix(result, "\n")
+ *s.out = append(*s.out, result)
+ return len(p), nil
+}
+
+// Check two directories
+func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+ srcFs, err := rc.GetFsNamed(ctx, in, "srcFs")
+ if err != nil && !rc.IsErrParamNotFound(err) {
+ return nil, err
+ }
+
+ dstFs, err := rc.GetFsNamed(ctx, in, "dstFs")
+ if err != nil {
+ return nil, err
+ }
+
+ checkFileFs, checkFileRemote, err := rc.GetFsAndRemoteNamed(ctx, in, "checkFileFs", "checkFileRemote")
+ if err != nil && !rc.IsErrParamNotFound(err) {
+ return nil, err
+ }
+
+ checkFileHash, err := in.GetString("checkFileHash")
+ if err != nil && !rc.IsErrParamNotFound(err) {
+ return nil, err
+ }
+
+ checkFileSet := 0
+ if checkFileHash != "" {
+ checkFileSet++
+ }
+ if checkFileFs != nil {
+ checkFileSet++
+ }
+ if checkFileRemote != "" {
+ checkFileSet++
+ }
+ if checkFileSet > 0 && checkFileSet < 3 {
+ return nil, fmt.Errorf("need all of checkFileFs, checkFileRemote, checkFileHash to be set together")
+ }
+
+ var checkFileHashType hash.Type
+ if checkFileHash != "" {
+ if err := checkFileHashType.Set(checkFileHash); err != nil {
+ return nil, err
+ }
+ if srcFs != nil {
+ return nil, rc.NewErrParamInvalid(errors.New("only supply dstFs when using checkFileHash"))
+ }
+ } else if srcFs == nil {
+ return nil, rc.NewErrParamInvalid(errors.New("need srcFs parameter when not using checkFileHash"))
+ }
+
+ oneway, _ := in.GetBool("oneWay")
+ download, _ := in.GetBool("download")
+
+ opt := &CheckOpt{
+ Fsrc: srcFs,
+ Fdst: dstFs,
+ OneWay: oneway,
+ }
+
+ out = rc.Params{}
+
+ getOutput := func(name string, Default bool) io.Writer {
+ active, err := in.GetBool(name)
+ if err != nil {
+ active = Default
+ }
+ if !active {
+ return nil
+ }
+ result := []string{}
+ out[name] = &result
+ return stringWriter{&result}
+ }
+
+ opt.Combined = getOutput("combined", false)
+ opt.MissingOnSrc = getOutput("missingOnSrc", true)
+ opt.MissingOnDst = getOutput("missingOnDst", true)
+ opt.Match = getOutput("match", false)
+ opt.Differ = getOutput("differ", true)
+ opt.Error = getOutput("error", true)
+
+ if checkFileHash != "" {
+ out["hashType"] = checkFileHashType.String()
+ err = CheckSum(ctx, dstFs, checkFileFs, checkFileRemote, checkFileHashType, opt, download)
+ } else {
+ if download {
+ err = CheckDownload(ctx, opt)
+ } else {
+ out["hashType"] = srcFs.Hashes().Overlap(dstFs.Hashes()).GetOne().String()
+ err = Check(ctx, opt)
+ }
+ }
+ if err != nil {
+ out["status"] = err.Error()
+ out["success"] = false
+ } else {
+ out["status"] = "OK"
+ out["success"] = true
+ }
+ return out, nil
+}
+
+func init() {
+ rc.Add(rc.Call{
+ Path: "operations/hashsum",
+ AuthRequired: true,
+ Fn: rcHashsum,
+ Title: "Produces a hashsum file for all the objects in the path.",
+ Help: `Produces a hash file for all the objects in the path using the hash
+named. The output is in the same format as the standard
+md5sum/sha1sum tool.
+
+This takes the following parameters:
+
+- fs - a remote name string e.g. "drive:" for the source, "/" for local filesystem
+ - this can point to a file and just that file will be returned in the listing.
+- hashType - type of hash to be used
+- download - check by downloading rather than with hash (boolean)
+- base64 - output the hashes in base64 rather than hex (boolean)
+
+If you supply the download flag, it will download the data from the
+remote and create the hash on the fly. This can be useful for remotes
+that don't support the given hash or if you really want to check all
+the data.
+
+Note that if you wish to supply a checkfile to check hashes against
+the current files then you should use operations/check instead of
+operations/hashsum.
+
+Returns:
+
+- hashsum - array of strings of the hashes
+- hashType - type of hash used
+
+Example:
+
+ $ rclone rc --loopback operations/hashsum fs=bin hashType=MD5 download=true base64=true
+ {
+ "hashType": "md5",
+ "hashsum": [
+ "WTSVLpuiXyJO_kGzJerRLg== backend-versions.sh",
+ "v1b_OlWCJO9LtNq3EIKkNQ== bisect-go-rclone.sh",
+ "VHbmHzHh4taXzgag8BAIKQ== bisect-rclone.sh",
+ ]
+ }
+
+See the [hashsum](/commands/rclone_hashsum/) command for more information on the above.
+`,
+ })
+}
+
+// Parse download, base64 and hashType parameters
+func parseHashParameters(in rc.Params) (download bool, base64 bool, ht hash.Type, err error) {
+ download, _ = in.GetBool("download")
+ base64, _ = in.GetBool("base64")
+ hashType, err := in.GetString("hashType")
+ if err != nil {
+ return
+ }
+ err = ht.Set(hashType)
+ return
+}
+
+// Hashsum a directory
+func rcHashsum(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+ ctx, f, err := rc.GetFsNamedFileOK(ctx, in, "fs")
+ if err != nil {
+ return nil, err
+ }
+
+ download, base64, ht, err := parseHashParameters(in)
+ if err != nil {
+ return out, err
+ }
+
+ hashes := []string{}
+ err = HashLister(ctx, ht, base64, download, f, stringWriter{&hashes})
+ out = rc.Params{
+ "hashType": ht.String(),
+ "hashsum": hashes,
+ }
+ return out, err
+}
+
+func init() {
+ rc.Add(rc.Call{
+ Path: "operations/hashsumfile",
+ AuthRequired: true,
+ Fn: rcHashsumFile,
+ Title: "Produces a hash for a single file.",
+ Help: `Produces a hash for a single file using the hash named.
+
+This takes the following parameters:
+
+- fs - a remote name string e.g. "drive:"
+- remote - a path within that remote e.g. "file.txt"
+- hashType - type of hash to be used
+- download - check by downloading rather than with hash (boolean)
+- base64 - output the hashes in base64 rather than hex (boolean)
+
+If you supply the download flag, it will download the data from the
+remote and create the hash on the fly. This can be useful for remotes
+that don't support the given hash or if you really want to read all
+the data.
+
+Returns:
+
+- hash - hash for the file
+- hashType - type of hash used
+
+Example:
+
+ $ rclone rc --loopback operations/hashsumfile fs=/ remote=/bin/bash hashType=MD5 download=true base64=true
+ {
+ "hashType": "md5",
+ "hash": "MDMw-fG2YXs7Uz5Nz-H68A=="
+ }
+
+See the [hashsum](/commands/rclone_hashsum/) command for more information on the above.
+`,
+ })
+}
+
+// Hashsum a file
+func rcHashsumFile(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+ f, remote, err := rc.GetFsAndRemote(ctx, in)
+ if err != nil {
+ return nil, err
+ }
+ download, base64, ht, err := parseHashParameters(in)
+ if err != nil {
+ return out, err
+ }
+ o, err := f.NewObject(ctx, remote)
+ if err != nil {
+ return nil, err
+ }
+ sum, err := HashSum(ctx, ht, base64, download, o)
+ out = rc.Params{
+ "hashType": ht.String(),
+ "hash": sum,
+ }
+ return out, err
+}
diff --git a/fs/operations/rc_test.go b/fs/operations/rc_test.go
new file mode 100644
index 0000000..04357fb
--- /dev/null
+++ b/fs/operations/rc_test.go
@@ -0,0 +1,892 @@
+package operations_test
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "os"
+ "path"
+ "sort"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/cache"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/fs/rc"
+ "github.com/rclone/rclone/fstest"
+ "github.com/rclone/rclone/lib/diskusage"
+ "github.com/rclone/rclone/lib/rest"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func rcNewRun(t *testing.T, method string) (*fstest.Run, *rc.Call) {
+ if *fstest.RemoteName != "" {
+ t.Skip("Skipping test on non local remote")
+ }
+ r := fstest.NewRun(t)
+ call := rc.Calls.Get(method)
+ assert.NotNil(t, call)
+ cache.Put(r.LocalName, r.Flocal)
+ cache.Put(r.FremoteName, r.Fremote)
+ return r, call
+}
+
+// operations/about: Return the space used on the remote
+func TestRcAbout(t *testing.T) {
+ r, call := rcNewRun(t, "operations/about")
+ r.Mkdir(context.Background(), r.Fremote)
+
+ // Will get an error if remote doesn't support About
+ expectedErr := r.Fremote.Features().About == nil
+
+ in := rc.Params{
+ "fs": r.FremoteName,
+ }
+ out, err := call.Fn(context.Background(), in)
+ if expectedErr {
+ assert.Error(t, err)
+ return
+ }
+ require.NoError(t, err)
+
+ // Can't really check the output much!
+ assert.NotEqual(t, int64(0), out["Total"])
+}
+
+// operations/cleanup: Remove trashed files in the remote or path
+func TestRcCleanup(t *testing.T) {
+ r, call := rcNewRun(t, "operations/cleanup")
+
+ in := rc.Params{
+ "fs": r.LocalName,
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.Error(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+ assert.Contains(t, err.Error(), "doesn't support cleanup")
+}
+
+// operations/copyfile: Copy a file from source remote to destination remote
+func TestRcCopyfile(t *testing.T) {
+ r, call := rcNewRun(t, "operations/copyfile")
+ file1 := r.WriteFile("file1", "file1 contents", t1)
+ r.Mkdir(context.Background(), r.Fremote)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t)
+
+ in := rc.Params{
+ "srcFs": r.LocalName,
+ "srcRemote": "file1",
+ "dstFs": r.FremoteName,
+ "dstRemote": "file1-renamed",
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+ r.CheckLocalItems(t, file1)
+ file1.Path = "file1-renamed"
+ r.CheckRemoteItems(t, file1)
+}
+
+// operations/copyurl: Copy the URL to the object
+func TestRcCopyurl(t *testing.T) {
+ r, call := rcNewRun(t, "operations/copyurl")
+ contents := "file1 contents\n"
+ file1 := r.WriteFile("file1", contents, t1)
+ r.Mkdir(context.Background(), r.Fremote)
+ r.CheckRemoteItems(t)
+
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, err := w.Write([]byte(contents))
+ assert.NoError(t, err)
+ }))
+ defer ts.Close()
+
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "remote": "file1",
+ "url": ts.URL,
+ "autoFilename": false,
+ "noClobber": false,
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+ in = rc.Params{
+ "fs": r.FremoteName,
+ "remote": "file1",
+ "url": ts.URL,
+ "autoFilename": false,
+ "noClobber": true,
+ }
+ out, err = call.Fn(context.Background(), in)
+ require.Error(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+ urlFileName := "filename.txt"
+ in = rc.Params{
+ "fs": r.FremoteName,
+ "remote": "",
+ "url": ts.URL + "/" + urlFileName,
+ "autoFilename": true,
+ "noClobber": false,
+ }
+ out, err = call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+ in = rc.Params{
+ "fs": r.FremoteName,
+ "remote": "",
+ "url": ts.URL,
+ "autoFilename": true,
+ "noClobber": false,
+ }
+ out, err = call.Fn(context.Background(), in)
+ require.Error(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported)
+}
+
+// operations/delete: Remove files in the path
+func TestRcDelete(t *testing.T) {
+ r, call := rcNewRun(t, "operations/delete")
+
+ file1 := r.WriteObject(context.Background(), "small", "1234567890", t2) // 10 bytes
+ file2 := r.WriteObject(context.Background(), "medium", "------------------------------------------------------------", t1) // 60 bytes
+ file3 := r.WriteObject(context.Background(), "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
+ r.CheckRemoteItems(t, file1, file2, file3)
+
+ in := rc.Params{
+ "fs": r.FremoteName,
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+ r.CheckRemoteItems(t)
+}
+
+// operations/deletefile: Remove the single file pointed to
+func TestRcDeletefile(t *testing.T) {
+ r, call := rcNewRun(t, "operations/deletefile")
+
+ file1 := r.WriteObject(context.Background(), "small", "1234567890", t2) // 10 bytes
+ file2 := r.WriteObject(context.Background(), "medium", "------------------------------------------------------------", t1) // 60 bytes
+ r.CheckRemoteItems(t, file1, file2)
+
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "remote": "small",
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+ r.CheckRemoteItems(t, file2)
+}
+
+// operations/list: List the given remote and path in JSON format.
+func TestRcList(t *testing.T) {
+ r, call := rcNewRun(t, "operations/list")
+
+ file1 := r.WriteObject(context.Background(), "a", "a", t1)
+ file2 := r.WriteObject(context.Background(), "subdir/b", "bb", t2)
+
+ r.CheckRemoteItems(t, file1, file2)
+
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "remote": "",
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+
+ list := out["list"].([]*operations.ListJSONItem)
+ assert.Equal(t, 2, len(list))
+
+ checkFile1 := func(got *operations.ListJSONItem) {
+ assert.WithinDuration(t, t1, got.ModTime.When, time.Second)
+ assert.Equal(t, "a", got.Path)
+ assert.Equal(t, "a", got.Name)
+ assert.Equal(t, int64(1), got.Size)
+ assert.Equal(t, "application/octet-stream", got.MimeType)
+ assert.Equal(t, false, got.IsDir)
+ }
+ checkFile1(list[0])
+
+ checkSubdir := func(got *operations.ListJSONItem) {
+ assert.Equal(t, "subdir", got.Path)
+ assert.Equal(t, "subdir", got.Name)
+ // assert.Equal(t, int64(-1), got.Size) // size can vary for directories
+ assert.Equal(t, "inode/directory", got.MimeType)
+ assert.Equal(t, true, got.IsDir)
+ }
+ checkSubdir(list[1])
+
+ in = rc.Params{
+ "fs": r.FremoteName,
+ "remote": "",
+ "opt": rc.Params{
+ "recurse": true,
+ },
+ }
+ out, err = call.Fn(context.Background(), in)
+ require.NoError(t, err)
+
+ list = out["list"].([]*operations.ListJSONItem)
+ assert.Equal(t, 3, len(list))
+ checkFile1(list[0])
+ checkSubdir(list[1])
+
+ checkFile2 := func(got *operations.ListJSONItem) {
+ assert.WithinDuration(t, t2, got.ModTime.When, time.Second)
+ assert.Equal(t, "subdir/b", got.Path)
+ assert.Equal(t, "b", got.Name)
+ assert.Equal(t, int64(2), got.Size)
+ assert.Equal(t, "application/octet-stream", got.MimeType)
+ assert.Equal(t, false, got.IsDir)
+ }
+ checkFile2(list[2])
+}
+
+// operations/stat: Stat the given remote and path in JSON format.
+func TestRcStat(t *testing.T) {
+ r, call := rcNewRun(t, "operations/stat")
+
+ file1 := r.WriteObject(context.Background(), "subdir/a", "a", t1)
+
+ r.CheckRemoteItems(t, file1)
+
+ fetch := func(t *testing.T, remotePath string) *operations.ListJSONItem {
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "remote": remotePath,
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ return out["item"].(*operations.ListJSONItem)
+ }
+
+ t.Run("Root", func(t *testing.T) {
+ stat := fetch(t, "")
+ assert.Equal(t, "", stat.Path)
+ assert.Equal(t, "", stat.Name)
+ assert.Equal(t, int64(-1), stat.Size)
+ assert.Equal(t, "inode/directory", stat.MimeType)
+ assert.Equal(t, true, stat.IsDir)
+ })
+
+ t.Run("File", func(t *testing.T) {
+ stat := fetch(t, "subdir/a")
+ assert.WithinDuration(t, t1, stat.ModTime.When, time.Second)
+ assert.Equal(t, "subdir/a", stat.Path)
+ assert.Equal(t, "a", stat.Name)
+ assert.Equal(t, int64(1), stat.Size)
+ assert.Equal(t, "application/octet-stream", stat.MimeType)
+ assert.Equal(t, false, stat.IsDir)
+ })
+
+ t.Run("Subdir", func(t *testing.T) {
+ stat := fetch(t, "subdir")
+ assert.Equal(t, "subdir", stat.Path)
+ assert.Equal(t, "subdir", stat.Name)
+ // assert.Equal(t, int64(-1), stat.Size) // size can vary for directories
+ assert.Equal(t, "inode/directory", stat.MimeType)
+ assert.Equal(t, true, stat.IsDir)
+ })
+
+ t.Run("NotFound", func(t *testing.T) {
+ stat := fetch(t, "notfound")
+ assert.Nil(t, stat)
+ })
+}
+
+// operations/settier: Set the storage tier of a fs
+func TestRcSetTier(t *testing.T) {
+ ctx := context.Background()
+ r, call := rcNewRun(t, "operations/settier")
+ if !r.Fremote.Features().SetTier {
+ t.Skip("settier not supported")
+ }
+ file1 := r.WriteObject(context.Background(), "file1", "file1 contents", t1)
+ r.CheckRemoteItems(t, file1)
+
+ // Because we don't know what the current tier options here are, let's
+ // just get the current tier, and reuse that
+ o, err := r.Fremote.NewObject(ctx, file1.Path)
+ require.NoError(t, err)
+ trr, ok := o.(fs.GetTierer)
+ require.True(t, ok)
+ ctier := trr.GetTier()
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "tier": ctier,
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+}
+
+// operations/settier: Set the storage tier of a file
+func TestRcSetTierFile(t *testing.T) {
+ ctx := context.Background()
+ r, call := rcNewRun(t, "operations/settierfile")
+ if !r.Fremote.Features().SetTier {
+ t.Skip("settier not supported")
+ }
+ file1 := r.WriteObject(context.Background(), "file1", "file1 contents", t1)
+ r.CheckRemoteItems(t, file1)
+
+ // Because we don't know what the current tier options here are, let's
+ // just get the current tier, and reuse that
+ o, err := r.Fremote.NewObject(ctx, file1.Path)
+ require.NoError(t, err)
+ trr, ok := o.(fs.GetTierer)
+ require.True(t, ok)
+ ctier := trr.GetTier()
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "remote": "file1",
+ "tier": ctier,
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+}
+
+// operations/mkdir: Make a destination directory or container
+func TestRcMkdir(t *testing.T) {
+ ctx := context.Background()
+ r, call := rcNewRun(t, "operations/mkdir")
+ r.Mkdir(context.Background(), r.Fremote)
+
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote))
+
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "remote": "subdir",
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(ctx, r.Fremote))
+}
+
+// operations/movefile: Move a file from source remote to destination remote
+func TestRcMovefile(t *testing.T) {
+ r, call := rcNewRun(t, "operations/movefile")
+ file1 := r.WriteFile("file1", "file1 contents", t1)
+ r.Mkdir(context.Background(), r.Fremote)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t)
+
+ in := rc.Params{
+ "srcFs": r.LocalName,
+ "srcRemote": "file1",
+ "dstFs": r.FremoteName,
+ "dstRemote": "file1-renamed",
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+ r.CheckLocalItems(t)
+ file1.Path = "file1-renamed"
+ r.CheckRemoteItems(t, file1)
+}
+
+// operations/purge: Remove a directory or container and all of its contents
+func TestRcPurge(t *testing.T) {
+ ctx := context.Background()
+ r, call := rcNewRun(t, "operations/purge")
+ file1 := r.WriteObject(context.Background(), "subdir/file1", "subdir/file1 contents", t1)
+
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"subdir"}, fs.GetModifyWindow(ctx, r.Fremote))
+
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "remote": "subdir",
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote))
+}
+
+// operations/rmdir: Remove an empty directory or container
+func TestRcRmdir(t *testing.T) {
+ ctx := context.Background()
+ r, call := rcNewRun(t, "operations/rmdir")
+ r.Mkdir(context.Background(), r.Fremote)
+ assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir"))
+
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(ctx, r.Fremote))
+
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "remote": "subdir",
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote))
+}
+
+// operations/rmdirs: Remove all the empty directories in the path
+func TestRcRmdirs(t *testing.T) {
+ ctx := context.Background()
+ r, call := rcNewRun(t, "operations/rmdirs")
+ r.Mkdir(context.Background(), r.Fremote)
+ assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir"))
+ assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir/subsubdir"))
+
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir", "subdir/subsubdir"}, fs.GetModifyWindow(ctx, r.Fremote))
+
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "remote": "subdir",
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote))
+
+ assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir"))
+ assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir/subsubdir"))
+
+ in = rc.Params{
+ "fs": r.FremoteName,
+ "remote": "subdir",
+ "leaveRoot": true,
+ }
+ out, err = call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Equal(t, rc.Params(nil), out)
+
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(ctx, r.Fremote))
+
+}
+
+// operations/size: Count the number of bytes and files in remote
+func TestRcSize(t *testing.T) {
+ r, call := rcNewRun(t, "operations/size")
+ file1 := r.WriteObject(context.Background(), "small", "1234567890", t2) // 10 bytes
+ file2 := r.WriteObject(context.Background(), "subdir/medium", "------------------------------------------------------------", t1) // 60 bytes
+ file3 := r.WriteObject(context.Background(), "subdir/subsubdir/large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 50 bytes
+ r.CheckRemoteItems(t, file1, file2, file3)
+
+ in := rc.Params{
+ "fs": r.FremoteName,
+ }
+ out, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ assert.Equal(t, rc.Params{
+ "count": int64(3),
+ "bytes": int64(120),
+ "sizeless": int64(0),
+ }, out)
+}
+
+// operations/publiclink: Create or retrieve a public link to the given file or folder.
+func TestRcPublicLink(t *testing.T) {
+ r, call := rcNewRun(t, "operations/publiclink")
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "remote": "",
+ "expire": "5m",
+ "unlink": false,
+ }
+ _, err := call.Fn(context.Background(), in)
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "doesn't support public links")
+}
+
+// operations/fsinfo: Return information about the remote
+func TestRcFsInfo(t *testing.T) {
+ r, call := rcNewRun(t, "operations/fsinfo")
+ in := rc.Params{
+ "fs": r.FremoteName,
+ }
+ got, err := call.Fn(context.Background(), in)
+ require.NoError(t, err)
+ want := operations.GetFsInfo(r.Fremote)
+ assert.Equal(t, want.Name, got["Name"])
+ assert.Equal(t, want.Root, got["Root"])
+ assert.Equal(t, want.String, got["String"])
+ assert.Equal(t, float64(want.Precision), got["Precision"])
+ var hashes []any
+ for _, hash := range want.Hashes {
+ hashes = append(hashes, hash)
+ }
+ assert.Equal(t, hashes, got["Hashes"])
+ var features = map[string]any{}
+ for k, v := range want.Features {
+ features[k] = v
+ }
+ assert.Equal(t, features, got["Features"])
+
+}
+
+// operations/uploadfile : Tests if upload file succeeds
+func TestUploadFile(t *testing.T) {
+ r, call := rcNewRun(t, "operations/uploadfile")
+ ctx := context.Background()
+
+ testFileName := "uploadfile-test.txt"
+ testFileContent := "Hello World"
+ r.WriteFile(testFileName, testFileContent, t1)
+ testItem1 := fstest.NewItem(testFileName, testFileContent, t1)
+ testItem2 := fstest.NewItem(path.Join("subdir", testFileName), testFileContent, t1)
+
+ currentFile, err := os.Open(path.Join(r.LocalName, testFileName))
+ require.NoError(t, err)
+
+ defer func() {
+ assert.NoError(t, currentFile.Close())
+ }()
+
+ formReader, contentType, _, err := rest.MultipartUpload(ctx, currentFile, url.Values{}, "file", testFileName, "application/octet-stream")
+ require.NoError(t, err)
+
+ httpReq := httptest.NewRequest("POST", "/", formReader)
+ httpReq.Header.Add("Content-Type", contentType)
+
+ in := rc.Params{
+ "_request": httpReq,
+ "fs": r.FremoteName,
+ "remote": "",
+ }
+
+ _, err = call.Fn(context.Background(), in)
+ require.NoError(t, err)
+
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{testItem1}, nil, fs.ModTimeNotSupported)
+
+ assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir"))
+
+ currentFile2, err := os.Open(path.Join(r.LocalName, testFileName))
+ require.NoError(t, err)
+
+ defer func() {
+ assert.NoError(t, currentFile2.Close())
+ }()
+
+ formReader, contentType, _, err = rest.MultipartUpload(ctx, currentFile2, url.Values{}, "file", testFileName, "application/octet-stream")
+ require.NoError(t, err)
+
+ httpReq = httptest.NewRequest("POST", "/", formReader)
+ httpReq.Header.Add("Content-Type", contentType)
+
+ in = rc.Params{
+ "_request": httpReq,
+ "fs": r.FremoteName,
+ "remote": "subdir",
+ }
+
+ _, err = call.Fn(context.Background(), in)
+ require.NoError(t, err)
+
+ fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{testItem1, testItem2}, nil, fs.ModTimeNotSupported)
+
+}
+
+// operations/command: Runs a backend command
+func TestRcCommand(t *testing.T) {
+ r, call := rcNewRun(t, "backend/command")
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "command": "noop",
+ "opt": map[string]string{
+ "echo": "true",
+ "blue": "",
+ },
+ "arg": []string{
+ "path1",
+ "path2",
+ },
+ }
+ got, err := call.Fn(context.Background(), in)
+ if err != nil {
+ assert.False(t, r.Fremote.Features().IsLocal, "mustn't fail on local remote")
+ assert.Contains(t, err.Error(), "command not found")
+ return
+ }
+ want := rc.Params{"result": map[string]any{
+ "arg": []string{
+ "path1",
+ "path2",
+ },
+ "name": "noop",
+ "opt": map[string]string{
+ "blue": "",
+ "echo": "true",
+ },
+ }}
+ assert.Equal(t, want, got)
+ errTxt := "explosion in the sausage factory"
+ in["opt"].(map[string]string)["error"] = errTxt
+ _, err = call.Fn(context.Background(), in)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), errTxt)
+}
+
+// operations/command: Runs a backend command
+func TestRcDu(t *testing.T) {
+ ctx := context.Background()
+ _, call := rcNewRun(t, "core/du")
+ in := rc.Params{}
+ out, err := call.Fn(ctx, in)
+ if err == diskusage.ErrUnsupported {
+ t.Skip(err)
+ }
+ assert.NotEqual(t, "", out["dir"])
+ info := out["info"].(diskusage.Info)
+ assert.True(t, info.Total != 0)
+ assert.True(t, info.Total > info.Free)
+ assert.True(t, info.Total > info.Available)
+ assert.True(t, info.Free >= info.Available)
+}
+
+// operations/check: check the source and destination are the same
+func TestRcCheck(t *testing.T) {
+ ctx := context.Background()
+ r, call := rcNewRun(t, "operations/check")
+ r.Mkdir(ctx, r.Fremote)
+
+ MD5SUMS := `
+0ef726ce9b1a7692357ff70dd321d595 file1
+deadbeefcafe00000000000000000000 subdir/file2
+0386a8b8fcf672c326845c00ba41b9e2 subdir/subsubdir/file4
+`
+
+ file1 := r.WriteBoth(ctx, "file1", "file1 contents", t1)
+ file2 := r.WriteFile("subdir/file2", MD5SUMS, t2)
+ file3 := r.WriteObject(ctx, "subdir/subsubdir/file3", "file3 contents", t3)
+ file4a := r.WriteFile("subdir/subsubdir/file4", "file4 contents", t3)
+ file4b := r.WriteObject(ctx, "subdir/subsubdir/file4", "file4 different contents", t3)
+ // operations.HashLister(ctx, hash.MD5, false, false, r.Fremote, os.Stdout)
+
+ r.CheckLocalItems(t, file1, file2, file4a)
+ r.CheckRemoteItems(t, file1, file3, file4b)
+
+ pstring := func(items ...fstest.Item) *[]string {
+ xs := make([]string, len(items))
+ for i, item := range items {
+ xs[i] = item.Path
+ }
+ return &xs
+ }
+
+ for _, testName := range []string{"Normal", "Download"} {
+ t.Run(testName, func(t *testing.T) {
+ in := rc.Params{
+ "srcFs": r.LocalName,
+ "dstFs": r.FremoteName,
+ "combined": true,
+ "missingOnSrc": true,
+ "missingOnDst": true,
+ "match": true,
+ "differ": true,
+ "error": true,
+ }
+ if testName == "Download" {
+ in["download"] = true
+ }
+ out, err := call.Fn(ctx, in)
+ require.NoError(t, err)
+
+ combined := []string{
+ "= " + file1.Path,
+ "+ " + file2.Path,
+ "- " + file3.Path,
+ "* " + file4a.Path,
+ }
+ sort.Strings(combined)
+ sort.Strings(*out["combined"].(*[]string))
+ want := rc.Params{
+ "missingOnSrc": pstring(file3),
+ "missingOnDst": pstring(file2),
+ "differ": pstring(file4a),
+ "error": pstring(),
+ "match": pstring(file1),
+ "combined": &combined,
+ "status": "3 differences found",
+ "success": false,
+ }
+ if testName == "Normal" {
+ want["hashType"] = "md5"
+ }
+
+ assert.Equal(t, want, out)
+ })
+ }
+
+ t.Run("CheckFile", func(t *testing.T) {
+ // The checksum file is treated as the source and srcFs is not used
+ in := rc.Params{
+ "dstFs": r.FremoteName,
+ "combined": true,
+ "missingOnSrc": true,
+ "missingOnDst": true,
+ "match": true,
+ "differ": true,
+ "error": true,
+ "checkFileFs": r.LocalName,
+ "checkFileRemote": file2.Path,
+ "checkFileHash": "md5",
+ }
+ out, err := call.Fn(ctx, in)
+ require.NoError(t, err)
+
+ combined := []string{
+ "= " + file1.Path,
+ "+ " + file2.Path,
+ "- " + file3.Path,
+ "* " + file4a.Path,
+ }
+ sort.Strings(combined)
+ sort.Strings(*out["combined"].(*[]string))
+ if strings.HasPrefix(out["status"].(string), "file not in") {
+ out["status"] = "file not in"
+ }
+ want := rc.Params{
+ "missingOnSrc": pstring(file3),
+ "missingOnDst": pstring(file2),
+ "differ": pstring(file4a),
+ "error": pstring(),
+ "match": pstring(file1),
+ "combined": &combined,
+ "hashType": "md5",
+ "status": "file not in",
+ "success": false,
+ }
+
+ assert.Equal(t, want, out)
+ })
+
+}
+
+// operations/hashsum: hashsum a directory
+func TestRcHashsum(t *testing.T) {
+ ctx := context.Background()
+ r, call := rcNewRun(t, "operations/hashsum")
+ r.Mkdir(ctx, r.Fremote)
+
+ file1Contents := "file1 contents"
+ file1 := r.WriteBoth(ctx, "hashsum-file1", file1Contents, t1)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, file1)
+
+ hasher := hash.NewMultiHasher()
+ _, err := hasher.Write([]byte(file1Contents))
+ require.NoError(t, err)
+
+ for _, test := range []struct {
+ ht hash.Type
+ base64 bool
+ download bool
+ }{
+ {
+ ht: r.Fremote.Hashes().GetOne(),
+ }, {
+ ht: r.Fremote.Hashes().GetOne(),
+ base64: true,
+ }, {
+ ht: hash.Whirlpool,
+ base64: false,
+ download: true,
+ }, {
+ ht: hash.Whirlpool,
+ base64: true,
+ download: true,
+ },
+ } {
+ t.Run(fmt.Sprintf("hash=%v,base64=%v,download=%v", test.ht, test.base64, test.download), func(t *testing.T) {
+ file1Hash, err := hasher.SumString(test.ht, test.base64)
+ require.NoError(t, err)
+
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "hashType": test.ht.String(),
+ "base64": test.base64,
+ "download": test.download,
+ }
+
+ out, err := call.Fn(ctx, in)
+ require.NoError(t, err)
+ assert.Equal(t, test.ht.String(), out["hashType"])
+ want := []string{
+ fmt.Sprintf("%s hashsum-file1", file1Hash),
+ }
+ assert.Equal(t, want, out["hashsum"])
+ })
+ }
+}
+
+// operations/hashsum: hashsum a single file
+func TestRcHashsumSingleFile(t *testing.T) {
+ ctx := context.Background()
+ r, call := rcNewRun(t, "operations/hashsum")
+ r.Mkdir(ctx, r.Fremote)
+
+ file1Contents := "file1 contents"
+ file1 := r.WriteBoth(ctx, "hashsum-file1", file1Contents, t1)
+ file2Contents := "file2 contents"
+ file2 := r.WriteBoth(ctx, "hashsum-file2", file2Contents, t1)
+ r.CheckLocalItems(t, file1, file2)
+ r.CheckRemoteItems(t, file1, file2)
+
+ // Make an fs pointing to just the file
+ fsString := path.Join(r.FremoteName, file1.Path)
+
+ in := rc.Params{
+ "fs": fsString,
+ "hashType": "MD5",
+ "download": true,
+ }
+
+ out, err := call.Fn(ctx, in)
+ require.NoError(t, err)
+ assert.Equal(t, "md5", out["hashType"])
+ assert.Equal(t, []string{"0ef726ce9b1a7692357ff70dd321d595 hashsum-file1"}, out["hashsum"])
+}
+
+// operations/hashsumfile: hashsum a single file
+func TestRcHashsumFile(t *testing.T) {
+ ctx := context.Background()
+ r, call := rcNewRun(t, "operations/hashsumfile")
+ r.Mkdir(ctx, r.Fremote)
+
+ file1Contents := "file1 contents"
+ file1 := r.WriteBoth(ctx, "hashsumfile-file1", file1Contents, t1)
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, file1)
+
+ in := rc.Params{
+ "fs": r.FremoteName,
+ "remote": file1.Path,
+ "hashType": "MD5",
+ "download": true,
+ }
+
+ out, err := call.Fn(ctx, in)
+ require.NoError(t, err)
+ assert.Equal(t, "md5", out["hashType"])
+ assert.Equal(t, "0ef726ce9b1a7692357ff70dd321d595", out["hash"])
+}
diff --git a/fs/operations/reopen.go b/fs/operations/reopen.go
new file mode 100644
index 0000000..fcc2a68
--- /dev/null
+++ b/fs/operations/reopen.go
@@ -0,0 +1,346 @@
+package operations
+
+import (
+ "context"
+ "errors"
+ "io"
+ "sync"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/fserrors"
+)
+
+// AccountFn is a function which will be called after every read
+// from the ReOpen.
+//
+// It may return an error which will be passed back to the user.
+type AccountFn func(n int) error
+
+// ReOpen is a wrapper for an object reader which reopens the stream on error
+type ReOpen struct {
+ ctx context.Context
+ mu sync.Mutex // mutex to protect the below
+ readAtMu sync.Mutex // mutex to serialize the ReadAt calls
+ src fs.Object // object to open
+ baseOptions []fs.OpenOption // options to pass to initial open and where offset == 0
+ options []fs.OpenOption // option to pass on subsequent opens where offset != 0
+ rangeOption fs.RangeOption // adjust this range option on re-opens
+ rc io.ReadCloser // underlying stream
+ size int64 // total size of object - can be -ve
+ start int64 // absolute position to start reading from
+ end int64 // absolute position to end reading (exclusive)
+ offset int64 // offset in the file we are at, offset from start
+ newOffset int64 // if different to offset, reopen needed
+ maxTries int // maximum number of retries
+ tries int // number of retries we've had so far in this stream
+ err error // if this is set then Read/Close calls will return it
+ opened bool // if set then rc is valid and needs closing
+ account AccountFn // account for a read
+ reads int // count how many times the data has been read
+ accountOn int // only account on or after this read
+}
+
+var (
+ errFileClosed = errors.New("file already closed")
+ errTooManyTries = errors.New("failed to reopen: too many retries")
+ errInvalidWhence = errors.New("reopen Seek: invalid whence")
+ errNegativeSeek = errors.New("reopen Seek: negative position")
+ errSeekPastEnd = errors.New("reopen Seek: attempt to seek past end of data")
+ errBadEndSeek = errors.New("reopen Seek: can't seek from end with unknown sized object")
+)
+
+// NewReOpen makes a handle which will reopen itself and seek to where
+// it was on errors up to maxTries times.
+//
+// If an fs.HashesOption is set this will be applied when reading from
+// the start.
+//
+// If an fs.RangeOption is set then this will applied when reading from
+// the start, and updated on retries.
+func NewReOpen(ctx context.Context, src fs.Object, maxTries int, options ...fs.OpenOption) (rc *ReOpen, err error) {
+ h := &ReOpen{
+ ctx: ctx,
+ src: src,
+ maxTries: maxTries,
+ baseOptions: options,
+ size: src.Size(),
+ start: 0,
+ offset: 0,
+ newOffset: -1, // -1 means no seek required
+ }
+ h.mu.Lock()
+ defer h.mu.Unlock()
+
+ // Filter the options for subsequent opens
+ h.options = make([]fs.OpenOption, 0, len(options)+1)
+ var limit int64 = -1
+ for _, option := range options {
+ switch x := option.(type) {
+ case *fs.HashesOption:
+ // leave hash option out when ranging
+ case *fs.RangeOption:
+ h.start, limit = x.Decode(h.end)
+ case *fs.SeekOption:
+ h.start, limit = x.Offset, -1
+ default:
+ h.options = append(h.options, option)
+ }
+ }
+
+ // Put our RangeOption on the end
+ h.rangeOption.Start = h.start
+ h.options = append(h.options, &h.rangeOption)
+
+ // If a size range is set then set the end point of the file to that
+ if limit >= 0 && h.size >= 0 {
+ h.end = h.start + limit
+ h.rangeOption.End = h.end - 1 // remember range options are inclusive
+ } else {
+ h.end = h.size
+ h.rangeOption.End = -1
+ }
+
+ err = h.open()
+ if err != nil {
+ return nil, err
+ }
+ return h, nil
+}
+
+// Open makes a handle which will reopen itself and seek to where it
+// was on errors.
+//
+// If an fs.HashesOption is set this will be applied when reading from
+// the start.
+//
+// If an fs.RangeOption is set then this will applied when reading from
+// the start, and updated on retries.
+//
+// It will obey LowLevelRetries in the ctx as the maximum number of
+// tries.
+//
+// Use this instead of calling the Open method on fs.Objects
+func Open(ctx context.Context, src fs.Object, options ...fs.OpenOption) (rc *ReOpen, err error) {
+ maxTries := fs.GetConfig(ctx).LowLevelRetries
+ return NewReOpen(ctx, src, maxTries, options...)
+}
+
+// open the underlying handle - call with lock held
+//
+// we don't retry here as the Open() call will itself have low level retries
+func (h *ReOpen) open() error {
+ var opts []fs.OpenOption
+ if h.offset == 0 {
+ // if reading from the start using the initial options
+ opts = h.baseOptions
+ } else {
+ // otherwise use the filtered options
+ opts = h.options
+ // Adjust range start to where we have got to
+ h.rangeOption.Start = h.start + h.offset
+ }
+ // Make a copy of the options as fs.FixRangeOption modifies them :-(
+ opts = append(make([]fs.OpenOption, 0, len(opts)), opts...)
+ h.tries++
+ if h.tries > h.maxTries {
+ h.err = errTooManyTries
+ } else {
+ h.rc, h.err = h.src.Open(h.ctx, opts...)
+ }
+ if h.err != nil {
+ if h.tries > 1 {
+ fs.Debugf(h.src, "Reopen failed after offset %d bytes read: %v", h.offset, h.err)
+ }
+ return h.err
+ }
+ h.opened = true
+ return nil
+}
+
+// reopen the underlying handle by closing it and reopening it.
+func (h *ReOpen) reopen() (err error) {
+ // close underlying stream if needed
+ if h.opened {
+ h.opened = false
+ _ = h.rc.Close()
+ }
+ return h.open()
+}
+
+// account for n bytes being read
+func (h *ReOpen) accountRead(n int) error {
+ if h.account == nil {
+ return nil
+ }
+ // Don't start accounting until we've reached this many reads
+ //
+ // rw.reads will be 1 the first time this is called
+ // rw.accountOn 2 means start accounting on the 2nd read through
+ if h.reads >= h.accountOn {
+ return h.account(n)
+ }
+ return nil
+}
+
+// Read bytes retrying as necessary
+func (h *ReOpen) Read(p []byte) (n int, err error) {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if h.err != nil {
+ // return a previous error if there is one
+ return n, h.err
+ }
+
+ // re-open if seek needed
+ if h.newOffset >= 0 {
+ if h.offset != h.newOffset {
+ fs.Debugf(h.src, "Seek from %d to %d", h.offset, h.newOffset)
+ h.offset = h.newOffset
+ err = h.reopen()
+ if err != nil {
+ return 0, err
+ }
+ }
+ h.newOffset = -1
+ }
+
+ // Read a full buffer
+ startOffset := h.offset
+ var nn int
+ for n < len(p) && err == nil {
+ nn, err = h.rc.Read(p[n:])
+ n += nn
+ h.offset += int64(nn)
+ if err != nil && err != io.EOF {
+ h.err = err
+ if !fserrors.IsNoLowLevelRetryError(err) {
+ fs.Debugf(h.src, "Reopening on read failure after offset %d bytes: retry %d/%d: %v", h.offset, h.tries, h.maxTries, err)
+ if h.reopen() == nil {
+ err = nil
+ }
+ }
+ }
+ }
+ // Count a read of the data if we read from the start successfully
+ if startOffset == 0 && n != 0 {
+ h.reads++
+ }
+ // Account the read
+ accErr := h.accountRead(n)
+ if err == nil {
+ err = accErr
+ }
+ return n, err
+}
+
+// ReadAt reads len(p) bytes at absolute offset off without changing
+// the read position.
+//
+// Note: operations are serialized; it won't behave like a truly
+// concurrent ReaderAt.
+func (h *ReOpen) ReadAt(p []byte, off int64) (n int, err error) {
+ h.readAtMu.Lock()
+ defer h.readAtMu.Unlock()
+
+ // Save current position
+ cur, err := h.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return 0, err
+ }
+ // Seek to requested offset
+ if _, err = h.Seek(off, io.SeekStart); err != nil {
+ return 0, err
+ }
+ // Restore position on exit
+ defer func() {
+ if _, seekErr := h.Seek(cur, io.SeekStart); seekErr != nil && err == nil {
+ err = seekErr
+ }
+ }()
+
+ // Fill p fully unless EOF
+ return h.Read(p)
+}
+
+// Seek sets the offset for the next Read or Write to offset, interpreted
+// according to whence: SeekStart means relative to the start of the file,
+// SeekCurrent means relative to the current offset, and SeekEnd means relative
+// to the end (for example, offset = -2 specifies the penultimate byte of the
+// file). Seek returns the new offset relative to the start of the file or an
+// error, if any.
+//
+// Seeking to an offset before the start of the file is an error. Seeking
+// to any positive offset may be allowed, but if the new offset exceeds the
+// size of the underlying object the behavior of subsequent I/O operations is
+// implementation-dependent.
+func (h *ReOpen) Seek(offset int64, whence int) (int64, error) {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if h.err != nil {
+ // return a previous error if there is one
+ return 0, h.err
+ }
+ var abs int64
+ var size = h.end - h.start
+ switch whence {
+ case io.SeekStart:
+ abs = offset
+ case io.SeekCurrent:
+ if h.newOffset >= 0 {
+ abs = h.newOffset + offset
+ } else {
+ abs = h.offset + offset
+ }
+ case io.SeekEnd:
+ if h.size < 0 {
+ return 0, errBadEndSeek
+ }
+ abs = size + offset
+ default:
+ return 0, errInvalidWhence
+ }
+ if abs < 0 {
+ return 0, errNegativeSeek
+ }
+ if h.size >= 0 && abs > size {
+ return size, errSeekPastEnd
+ }
+
+ h.tries = 0 // Reset open count on seek
+ h.newOffset = abs // New offset - applied in Read
+ return abs, nil
+}
+
+// Close the stream
+func (h *ReOpen) Close() error {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if !h.opened {
+ return errFileClosed
+ }
+ h.opened = false
+ h.err = errFileClosed
+ return h.rc.Close()
+}
+
+// SetAccounting should be provided with a function which will be
+// called after every read from the RW.
+//
+// It may return an error which will be passed back to the user.
+func (h *ReOpen) SetAccounting(account AccountFn) *ReOpen {
+ h.account = account
+ return h
+}
+
+// DelayAccounting makes sure the accounting function only gets called
+// on the i-th or later read of the data from this point (counting
+// from 1).
+//
+// This is useful so that we don't account initial reads of the data
+// e.g. when calculating hashes.
+//
+// Set this to 0 to account everything.
+func (h *ReOpen) DelayAccounting(i int) {
+ h.accountOn = i
+ h.reads = 0
+}
diff --git a/fs/operations/reopen_test.go b/fs/operations/reopen_test.go
new file mode 100644
index 0000000..19f6ca3
--- /dev/null
+++ b/fs/operations/reopen_test.go
@@ -0,0 +1,429 @@
+package operations
+
+import (
+ "context"
+ "errors"
+ "io"
+ "testing"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fstest/mockobject"
+ "github.com/rclone/rclone/lib/pool"
+ "github.com/rclone/rclone/lib/readers"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// check interfaces
+var (
+ _ io.ReadSeekCloser = (*ReOpen)(nil)
+ _ io.ReaderAt = (*ReOpen)(nil)
+ _ pool.DelayAccountinger = (*ReOpen)(nil)
+)
+
+var errorTestError = errors.New("test error")
+
+// this is a wrapper for a mockobject with a custom Open function
+//
+// breaks indicate the number of bytes to read before returning an
+// error
+type reOpenTestObject struct {
+ fs.Object
+ t *testing.T
+ wantStart int64
+ breaks []int64
+ unknownSize bool
+}
+
+// Open opens the file for read. Call Close() on the returned io.ReadCloser
+//
+// This will break after reading the number of bytes in breaks
+func (o *reOpenTestObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+ // Lots of backends do this - make sure it works as it modifies options
+ fs.FixRangeOption(options, o.Size())
+ gotHash := false
+ gotRange := false
+ startPos := int64(0)
+ for _, option := range options {
+ switch x := option.(type) {
+ case *fs.HashesOption:
+ gotHash = true
+ case *fs.RangeOption:
+ gotRange = true
+ startPos = x.Start
+ if o.unknownSize {
+ assert.Equal(o.t, int64(-1), x.End)
+ }
+ case *fs.SeekOption:
+ startPos = x.Offset
+ }
+ }
+ assert.Equal(o.t, o.wantStart, startPos)
+ // Check if ranging, mustn't have hash if offset != 0
+ if gotHash && gotRange {
+ assert.Equal(o.t, int64(0), startPos)
+ }
+ rc, err := o.Object.Open(ctx, options...)
+ if err != nil {
+ return nil, err
+ }
+ if len(o.breaks) > 0 {
+ // Pop a breakpoint off
+ N := o.breaks[0]
+ o.breaks = o.breaks[1:]
+ o.wantStart += N
+ // If 0 then return an error immediately
+ if N == 0 {
+ return nil, errorTestError
+ }
+ // Read N bytes then an error
+ r := io.MultiReader(&io.LimitedReader{R: rc, N: N}, readers.ErrorReader{Err: errorTestError})
+ // Wrap with Close in a new readCloser
+ rc = readCloser{Reader: r, Closer: rc}
+ }
+ return rc, nil
+}
+
+func TestReOpen(t *testing.T) {
+ for _, testName := range []string{"Normal", "WithRangeOption", "WithSeekOption", "UnknownSize"} {
+ t.Run(testName, func(t *testing.T) {
+ // Contents for the mock object
+ var (
+ reOpenTestcontents = []byte("0123456789")
+ expectedRead = reOpenTestcontents
+ rangeOption *fs.RangeOption
+ seekOption *fs.SeekOption
+ unknownSize = false
+ )
+ switch testName {
+ case "Normal":
+ case "WithRangeOption":
+ rangeOption = &fs.RangeOption{Start: 1, End: 7} // range is inclusive
+ expectedRead = reOpenTestcontents[1:8]
+ case "WithSeekOption":
+ seekOption = &fs.SeekOption{Offset: 2}
+ expectedRead = reOpenTestcontents[2:]
+ case "UnknownSize":
+ rangeOption = &fs.RangeOption{Start: 1, End: -1}
+ expectedRead = reOpenTestcontents[1:]
+ unknownSize = true
+ default:
+ panic("bad test name")
+ }
+
+ // Start the test with the given breaks
+ testReOpen := func(breaks []int64, maxRetries int) (*ReOpen, *reOpenTestObject, error) {
+ srcOrig := mockobject.New("potato").WithContent(reOpenTestcontents, mockobject.SeekModeNone)
+ srcOrig.SetUnknownSize(unknownSize)
+ src := &reOpenTestObject{
+ Object: srcOrig,
+ t: t,
+ breaks: breaks,
+ unknownSize: unknownSize,
+ }
+ opts := []fs.OpenOption{}
+ if rangeOption == nil && seekOption == nil {
+ opts = append(opts, &fs.HashesOption{Hashes: hash.NewHashSet(hash.MD5)})
+ }
+ if rangeOption != nil {
+ opts = append(opts, rangeOption)
+ src.wantStart = rangeOption.Start
+ }
+ if seekOption != nil {
+ opts = append(opts, seekOption)
+ src.wantStart = seekOption.Offset
+ }
+ rc, err := NewReOpen(context.Background(), src, maxRetries, opts...)
+ return rc, src, err
+ }
+
+ // Reset the start after a seek, taking into account the offset
+ setWantStart := func(src *reOpenTestObject, x int64) {
+ src.wantStart = x
+ if rangeOption != nil {
+ src.wantStart += rangeOption.Start
+ } else if seekOption != nil {
+ src.wantStart += seekOption.Offset
+ }
+ }
+
+ t.Run("Basics", func(t *testing.T) {
+ // open
+ h, _, err := testReOpen(nil, 10)
+ assert.NoError(t, err)
+
+ // Check contents read correctly
+ got, err := io.ReadAll(h)
+ assert.NoError(t, err)
+ assert.Equal(t, expectedRead, got)
+
+ // Check read after end
+ var buf = make([]byte, 1)
+ n, err := h.Read(buf)
+ assert.Equal(t, 0, n)
+ assert.Equal(t, io.EOF, err)
+
+ // Rewind the stream
+ _, err = h.Seek(0, io.SeekStart)
+ require.NoError(t, err)
+
+ // Check contents read correctly
+ got, err = io.ReadAll(h)
+ assert.NoError(t, err)
+ assert.Equal(t, expectedRead, got)
+
+ // Check close
+ assert.NoError(t, h.Close())
+
+ // Check double close
+ assert.Equal(t, errFileClosed, h.Close())
+
+ // Check read after close
+ n, err = h.Read(buf)
+ assert.Equal(t, 0, n)
+ assert.Equal(t, errFileClosed, err)
+ })
+
+ t.Run("ErrorAtStart", func(t *testing.T) {
+ // open with immediate breaking
+ h, _, err := testReOpen([]int64{0}, 10)
+ assert.Equal(t, errorTestError, err)
+ assert.Nil(t, h)
+ })
+
+ t.Run("WithErrors", func(t *testing.T) {
+ // open with a few break points but less than the max
+ h, _, err := testReOpen([]int64{2, 1, 3}, 10)
+ assert.NoError(t, err)
+
+ // check contents
+ got, err := io.ReadAll(h)
+ assert.NoError(t, err)
+ assert.Equal(t, expectedRead, got)
+
+ // check close
+ assert.NoError(t, h.Close())
+ })
+
+ t.Run("TooManyErrors", func(t *testing.T) {
+ // open with a few break points but >= the max
+ h, _, err := testReOpen([]int64{2, 1, 3}, 3)
+ assert.NoError(t, err)
+
+ // check contents
+ got, err := io.ReadAll(h)
+ assert.Equal(t, errorTestError, err)
+ assert.Equal(t, expectedRead[:6], got)
+
+ // check old error is returned
+ var buf = make([]byte, 1)
+ n, err := h.Read(buf)
+ assert.Equal(t, 0, n)
+ assert.Equal(t, errTooManyTries, err)
+
+ // Check close
+ assert.Equal(t, errFileClosed, h.Close())
+ })
+
+ t.Run("ReadAt", func(t *testing.T) {
+ // open
+ h, src, err := testReOpen([]int64{2, 1, 3}, 10)
+ assert.NoError(t, err)
+
+ buf := make([]byte, 5)
+
+ // Read at 0
+ n, err := h.ReadAt(buf, 0)
+ require.NoError(t, err)
+ assert.Equal(t, 5, n)
+ assert.Equal(t, expectedRead[:n], buf[:n])
+
+ // Read at 1
+ setWantStart(src, 1)
+ n, err = h.ReadAt(buf[:3], 1)
+ require.NoError(t, err)
+ assert.Equal(t, 3, n)
+ assert.Equal(t, expectedRead[1:n+1], buf[:n])
+
+ // check position unchanged
+ pos, err := h.Seek(0, io.SeekCurrent)
+ require.NoError(t, err)
+ assert.Equal(t, int64(0), pos)
+
+ // check close
+ assert.NoError(t, h.Close())
+ _, err = h.Seek(0, io.SeekCurrent)
+ assert.Equal(t, errFileClosed, err)
+ })
+
+ t.Run("Seek", func(t *testing.T) {
+ // open
+ h, src, err := testReOpen([]int64{2, 1, 3}, 10)
+ assert.NoError(t, err)
+
+ // Seek to end
+ pos, err := h.Seek(int64(len(expectedRead)), io.SeekStart)
+ assert.NoError(t, err)
+ assert.Equal(t, int64(len(expectedRead)), pos)
+
+ // Seek to start
+ pos, err = h.Seek(0, io.SeekStart)
+ assert.NoError(t, err)
+ assert.Equal(t, int64(0), pos)
+
+ // Should not allow seek past end
+ pos, err = h.Seek(int64(len(expectedRead))+1, io.SeekCurrent)
+ if !unknownSize {
+ assert.Equal(t, errSeekPastEnd, err)
+ assert.Equal(t, len(expectedRead), int(pos))
+ } else {
+ assert.Equal(t, nil, err)
+ assert.Equal(t, len(expectedRead)+1, int(pos))
+
+ // Seek back to start to get tests in sync
+ pos, err = h.Seek(0, io.SeekStart)
+ assert.NoError(t, err)
+ assert.Equal(t, int64(0), pos)
+ }
+
+ // Should not allow seek to negative position start
+ pos, err = h.Seek(-1, io.SeekCurrent)
+ assert.Equal(t, errNegativeSeek, err)
+ assert.Equal(t, 0, int(pos))
+
+ // Should not allow seek with invalid whence
+ pos, err = h.Seek(0, 3)
+ assert.Equal(t, errInvalidWhence, err)
+ assert.Equal(t, 0, int(pos))
+
+ // check read
+ dst := make([]byte, 5)
+ n, err := h.Read(dst)
+ assert.Nil(t, err)
+ assert.Equal(t, 5, n)
+ assert.Equal(t, expectedRead[:5], dst)
+
+ // Test io.SeekCurrent
+ pos, err = h.Seek(-3, io.SeekCurrent)
+ assert.Nil(t, err)
+ assert.Equal(t, 2, int(pos))
+
+ // check read
+ setWantStart(src, 2)
+ n, err = h.Read(dst)
+ assert.Nil(t, err)
+ assert.Equal(t, 5, n)
+ assert.Equal(t, expectedRead[2:7], dst)
+
+ pos, err = h.Seek(-2, io.SeekCurrent)
+ assert.Nil(t, err)
+ assert.Equal(t, 5, int(pos))
+
+ // Test io.SeekEnd
+ pos, err = h.Seek(-3, io.SeekEnd)
+ if !unknownSize {
+ assert.Nil(t, err)
+ assert.Equal(t, len(expectedRead)-3, int(pos))
+ } else {
+ assert.Equal(t, errBadEndSeek, err)
+ assert.Equal(t, 0, int(pos))
+
+ // sync
+ pos, err = h.Seek(1, io.SeekCurrent)
+ assert.Nil(t, err)
+ assert.Equal(t, 6, int(pos))
+ }
+
+ // check read
+ dst = make([]byte, 3)
+ setWantStart(src, int64(len(expectedRead)-3))
+ n, err = h.Read(dst)
+ assert.Nil(t, err)
+ assert.Equal(t, 3, n)
+ assert.Equal(t, expectedRead[len(expectedRead)-3:], dst)
+
+ // check close
+ assert.NoError(t, h.Close())
+ _, err = h.Seek(0, io.SeekCurrent)
+ assert.Equal(t, errFileClosed, err)
+ })
+
+ t.Run("AccountRead", func(t *testing.T) {
+ h, _, err := testReOpen(nil, 10)
+ assert.NoError(t, err)
+
+ var total int
+ h.SetAccounting(func(n int) error {
+ total += n
+ return nil
+ })
+
+ dst := make([]byte, 3)
+ n, err := h.Read(dst)
+ assert.Equal(t, 3, n)
+ assert.NoError(t, err)
+ assert.Equal(t, 3, total)
+ })
+
+ t.Run("AccountReadDelay", func(t *testing.T) {
+ h, _, err := testReOpen(nil, 10)
+ assert.NoError(t, err)
+
+ var total int
+ h.SetAccounting(func(n int) error {
+ total += n
+ return nil
+ })
+
+ rewind := func() {
+ _, err := h.Seek(0, io.SeekStart)
+ require.NoError(t, err)
+ }
+
+ h.DelayAccounting(3)
+
+ dst := make([]byte, 16)
+
+ n, err := h.Read(dst)
+ assert.Equal(t, len(expectedRead), n)
+ assert.Equal(t, io.EOF, err)
+ assert.Equal(t, 0, total)
+ rewind()
+
+ n, err = h.Read(dst)
+ assert.Equal(t, len(expectedRead), n)
+ assert.Equal(t, io.EOF, err)
+ assert.Equal(t, 0, total)
+ rewind()
+
+ n, err = h.Read(dst)
+ assert.Equal(t, len(expectedRead), n)
+ assert.Equal(t, io.EOF, err)
+ assert.Equal(t, len(expectedRead), total)
+ rewind()
+
+ n, err = h.Read(dst)
+ assert.Equal(t, len(expectedRead), n)
+ assert.Equal(t, io.EOF, err)
+ assert.Equal(t, 2*len(expectedRead), total)
+ rewind()
+ })
+
+ t.Run("AccountReadError", func(t *testing.T) {
+ // Test accounting errors
+ h, _, err := testReOpen(nil, 10)
+ assert.NoError(t, err)
+
+ h.SetAccounting(func(n int) error {
+ return errorTestError
+ })
+
+ dst := make([]byte, 3)
+ n, err := h.Read(dst)
+ assert.Equal(t, 3, n)
+ assert.Equal(t, errorTestError, err)
+ })
+ })
+ }
+}
generated by cgit v1.0 at 2026-05-27 06:49:42 +0000