diff options
30 files changed, 18351 insertions, 0 deletions
diff --git a/fs/license b/fs/license new file mode 100644 index 0000000..bc452e3 --- /dev/null +++ b/fs/license @@ -0,0 +1,19 @@ +Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/fs/operations/check.go b/fs/operations/check.go new file mode 100644 index 0000000..d4d1eb3 --- /dev/null +++ b/fs/operations/check.go @@ -0,0 +1,626 @@ +package operations + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "regexp" + "strings" + "sync" + "sync/atomic" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/march" + "github.com/rclone/rclone/lib/readers" + "golang.org/x/text/unicode/norm" +) + +// checkFn is the type of the checking function used in CheckFn() +// +// It should check the two objects (a, b) and return if they differ +// and whether the hash was used. +// +// If there are differences then this should Errorf the difference and +// the reason but return with err = nil. It should not CountError in +// this case. +type checkFn func(ctx context.Context, a, b fs.Object) (differ bool, noHash bool, err error) + +// CheckOpt contains options for the Check functions +type CheckOpt struct { + Fdst, Fsrc fs.Fs // fses to check + Check checkFn // function to use for checking + OneWay bool // one way only? + Combined io.Writer // a file with file names with leading sigils + MissingOnSrc io.Writer // files only in the destination + MissingOnDst io.Writer // files only in the source + Match io.Writer // matching files + Differ io.Writer // differing files + Error io.Writer // files with errors of some kind +} + +// checkMarch is used to march over two Fses in the same way as +// sync/copy +type checkMarch struct { + ctx context.Context + ioMu sync.Mutex + wg sync.WaitGroup + tokens chan struct{} + differences atomic.Int32 + noHashes atomic.Int32 + srcFilesMissing atomic.Int32 + dstFilesMissing atomic.Int32 + matches atomic.Int32 + opt CheckOpt +} + +// report outputs the fileName to out if required and to the combined log +func (c *checkMarch) report(o fs.DirEntry, out io.Writer, sigil rune) { + c.reportFilename(o.String(), out, sigil) +} + +func (c *checkMarch) reportFilename(filename string, out io.Writer, sigil rune) { + if out != nil { + SyncFprintf(out, "%s\n", filename) + } + if c.opt.Combined != nil { + SyncFprintf(c.opt.Combined, "%c %s\n", sigil, filename) + } +} + +// DstOnly have an object which is in the destination only +func (c *checkMarch) DstOnly(dst fs.DirEntry) (recurse bool) { + switch dst.(type) { + case fs.Object: + if c.opt.OneWay { + return false + } + err := fmt.Errorf("file not in %v", c.opt.Fsrc) + fs.Errorf(dst, "%v", err) + _ = fs.CountError(c.ctx, err) + c.differences.Add(1) + c.srcFilesMissing.Add(1) + c.report(dst, c.opt.MissingOnSrc, '-') + case fs.Directory: + // Do the same thing to the entire contents of the directory + if c.opt.OneWay { + return false + } + return true + default: + panic("Bad object in DirEntries") + } + return false +} + +// SrcOnly have an object which is in the source only +func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) { + switch src.(type) { + case fs.Object: + err := fmt.Errorf("file not in %v", c.opt.Fdst) + fs.Errorf(src, "%v", err) + _ = fs.CountError(c.ctx, err) + c.differences.Add(1) + c.dstFilesMissing.Add(1) + c.report(src, c.opt.MissingOnDst, '+') + case fs.Directory: + // Do the same thing to the entire contents of the directory + return true + default: + panic("Bad object in DirEntries") + } + return false +} + +// check to see if two objects are identical using the check function +func (c *checkMarch) checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) { + ci := fs.GetConfig(ctx) + tr := accounting.Stats(ctx).NewCheckingTransfer(src, "checking") + defer func() { + tr.Done(ctx, err) + }() + if sizeDiffers(ctx, src, dst) { + err = fmt.Errorf("sizes differ") + fs.Errorf(src, "%v", err) + return true, false, nil + } + if ci.SizeOnly { + return false, false, nil + } + return c.opt.Check(ctx, dst, src) +} + +// Match is called when src and dst are present, so sync src to dst +func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse bool) { + switch srcX := src.(type) { + case fs.Object: + dstX, ok := dst.(fs.Object) + if ok { + if SkipDestructive(ctx, src, "check") { + return false + } + c.wg.Add(1) + c.tokens <- struct{}{} // put a token to limit concurrency + go func() { + defer func() { + <-c.tokens // get the token back to free up a slot + c.wg.Done() + }() + differ, noHash, err := c.checkIdentical(ctx, dstX, srcX) + if err != nil { + fs.Errorf(src, "%v", err) + _ = fs.CountError(ctx, err) + c.report(src, c.opt.Error, '!') + } else if differ { + c.differences.Add(1) + err := errors.New("files differ") + // the checkFn has already logged the reason + _ = fs.CountError(ctx, err) + c.report(src, c.opt.Differ, '*') + } else { + c.matches.Add(1) + c.report(src, c.opt.Match, '=') + if noHash { + c.noHashes.Add(1) + fs.Debugf(dstX, "OK - could not check hash") + } else { + fs.Debugf(dstX, "OK") + } + } + }() + } else { + err := fmt.Errorf("is file on %v but directory on %v", c.opt.Fsrc, c.opt.Fdst) + fs.Errorf(src, "%v", err) + _ = fs.CountError(ctx, err) + c.differences.Add(1) + c.dstFilesMissing.Add(1) + c.report(src, c.opt.MissingOnDst, '+') + } + case fs.Directory: + // Do the same thing to the entire contents of the directory + _, ok := dst.(fs.Directory) + if ok { + return true + } + err := fmt.Errorf("is file on %v but directory on %v", c.opt.Fdst, c.opt.Fsrc) + fs.Errorf(dst, "%v", err) + _ = fs.CountError(ctx, err) + c.differences.Add(1) + c.srcFilesMissing.Add(1) + c.report(dst, c.opt.MissingOnSrc, '-') + + default: + panic("Bad object in DirEntries") + } + return false +} + +// CheckFn checks the files in fsrc and fdst according to Size and +// hash using checkFunction on each file to check the hashes. +// +// checkFunction sees if dst and src are identical +// +// it returns true if differences were found +// it also returns whether it couldn't be hashed +func CheckFn(ctx context.Context, opt *CheckOpt) error { + ci := fs.GetConfig(ctx) + if opt.Check == nil { + return errors.New("internal error: nil check function") + } + c := &checkMarch{ + ctx: ctx, + tokens: make(chan struct{}, ci.Checkers), + opt: *opt, + } + + // set up a march over fdst and fsrc + m := &march.March{ + Ctx: ctx, + Fdst: c.opt.Fdst, + Fsrc: c.opt.Fsrc, + Dir: "", + Callback: c, + NoTraverse: ci.NoTraverse, + NoUnicodeNormalization: ci.NoUnicodeNormalization, + } + fs.Debugf(c.opt.Fdst, "Waiting for checks to finish") + err := m.Run(ctx) + c.wg.Wait() // wait for background go-routines + + return c.reportResults(ctx, err) +} + +func (c *checkMarch) reportResults(ctx context.Context, err error) error { + if c.dstFilesMissing.Load() > 0 { + fs.Logf(c.opt.Fdst, "%d files missing", c.dstFilesMissing.Load()) + } + if c.srcFilesMissing.Load() > 0 { + entity := "files" + if c.opt.Fsrc == nil { + entity = "hashes" + } + fs.Logf(c.opt.Fsrc, "%d %s missing", c.srcFilesMissing.Load(), entity) + } + + fs.Logf(c.opt.Fdst, "%d differences found", c.differences.Load()) + if errs := accounting.Stats(ctx).GetErrors(); errs > 0 { + fs.Logf(c.opt.Fdst, "%d errors while checking", errs) + } + if c.noHashes.Load() > 0 { + fs.Logf(c.opt.Fdst, "%d hashes could not be checked", c.noHashes.Load()) + } + if c.matches.Load() > 0 { + fs.Logf(c.opt.Fdst, "%d matching files", c.matches.Load()) + } + if err != nil { + return err + } + if c.differences.Load() > 0 { + // Return an already counted error so we don't double count this error too + err = fserrors.FsError(fmt.Errorf("%d differences found", c.differences.Load())) + fserrors.Count(err) + return err + } + return nil +} + +// Check the files in fsrc and fdst according to Size and hash +func Check(ctx context.Context, opt *CheckOpt) error { + optCopy := *opt + optCopy.Check = func(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) { + same, ht, err := CheckHashes(ctx, src, dst) + if err != nil { + return true, false, err + } + if ht == hash.None { + return false, true, nil + } + if !same { + err = fmt.Errorf("%v differ", ht) + fs.Errorf(src, "%v", err) + return true, false, nil + } + return false, false, nil + } + + return CheckFn(ctx, &optCopy) +} + +// CheckEqualReaders checks to see if in1 and in2 have the same +// content when read. +// +// it returns true if no differences were found +func CheckEqualReaders(in1, in2 io.Reader) (equal bool, err error) { + const bufSize = 64 * 1024 + buf1 := make([]byte, bufSize) + buf2 := make([]byte, bufSize) + for { + n1, err1 := readers.ReadFill(in1, buf1) + n2, err2 := readers.ReadFill(in2, buf2) + // check errors + if err1 != nil && err1 != io.EOF { + return false, err1 + } else if err2 != nil && err2 != io.EOF { + return false, err2 + } + // err1 && err2 are nil or io.EOF here + // process the data + if n1 != n2 || !bytes.Equal(buf1[:n1], buf2[:n2]) { + return false, nil + } + // if both streams finished the we have finished + if err1 == io.EOF && err2 == io.EOF { + break + } + } + return true, nil +} + +// CheckIdenticalDownload checks to see if dst and src are identical +// by reading all their bytes if necessary. +// +// it returns true if no differences were found +func CheckIdenticalDownload(ctx context.Context, src, dst fs.Object) (equal bool, err error) { + ci := fs.GetConfig(ctx) + err = Retry(ctx, src, ci.LowLevelRetries, func() error { + equal, err = checkIdenticalDownload(ctx, src, dst) + return err + }) + return equal, err +} + +// Does the work for CheckIdenticalDownload +func checkIdenticalDownload(ctx context.Context, src, dst fs.Object) (equal bool, err error) { + var in1, in2 io.ReadCloser + in1, err = Open(ctx, dst) + if err != nil { + return false, fmt.Errorf("failed to open %q: %w", dst, err) + } + tr1 := accounting.Stats(ctx).NewTransfer(dst, nil) + defer func() { + tr1.Done(ctx, nil) // error handling is done by the caller + }() + in1 = tr1.Account(ctx, in1).WithBuffer() // account and buffer the transfer + + in2, err = Open(ctx, src) + if err != nil { + return false, fmt.Errorf("failed to open %q: %w", src, err) + } + tr2 := accounting.Stats(ctx).NewTransfer(dst, nil) + defer func() { + tr2.Done(ctx, nil) // error handling is done by the caller + }() + in2 = tr2.Account(ctx, in2).WithBuffer() // account and buffer the transfer + + // To assign err variable before defer. + equal, err = CheckEqualReaders(in1, in2) + return +} + +// CheckDownload checks the files in fsrc and fdst according to Size +// and the actual contents of the files. +func CheckDownload(ctx context.Context, opt *CheckOpt) error { + optCopy := *opt + optCopy.Check = func(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) { + same, err := CheckIdenticalDownload(ctx, src, dst) + if err != nil { + return true, true, fmt.Errorf("failed to download: %w", err) + } + if !same { + err = errors.New("contents differ") + fs.Errorf(src, "%v", err) + return true, false, nil + } + return false, false, nil + } + return CheckFn(ctx, &optCopy) +} + +// ApplyTransforms handles --no-unicode-normalization and --ignore-case-sync for CheckSum +// so that it matches behavior of Check (where it's handled by March) +func ApplyTransforms(ctx context.Context, s string) string { + ci := fs.GetConfig(ctx) + return ToNormal(s, !ci.NoUnicodeNormalization, ci.IgnoreCaseSync) +} + +// ToNormal normalizes case and unicode form and returns the transformed string. +// It is similar to ApplyTransforms but does not use a context. +// If normUnicode == true, s will be transformed to NFC. +// If normCase == true, s will be transformed to lowercase. +// If both are true, both transformations will be performed. +func ToNormal(s string, normUnicode, normCase bool) string { + if normUnicode { + s = norm.NFC.String(s) + } + if normCase { + s = strings.ToLower(s) + } + return s +} + +// CheckSum checks filesystem hashes against a SUM file +func CheckSum(ctx context.Context, fsrc, fsum fs.Fs, sumFile string, hashType hash.Type, opt *CheckOpt, download bool) error { + var options CheckOpt + if opt != nil { + options = *opt + } else { + // default options for hashsum -c + options.Combined = os.Stdout + } + // CheckSum treats Fsrc and Fdst specially: + options.Fsrc = nil // no file system here, corresponds to the sum list + options.Fdst = fsrc // denotes the file system to check + opt = &options // override supplied argument + + if !download && (hashType == hash.None || !opt.Fdst.Hashes().Contains(hashType)) { + return fmt.Errorf("%s: hash type is not supported by file system: %s", hashType, opt.Fdst) + } + + if sumFile == "" { + return fmt.Errorf("not a sum file: %s", fsum) + } + sumObj, err := fsum.NewObject(ctx, sumFile) + if err != nil { + return fmt.Errorf("cannot open sum file: %w", err) + } + hashes, err := ParseSumFile(ctx, sumObj) + if err != nil { + return fmt.Errorf("failed to parse sum file: %w", err) + } + + ci := fs.GetConfig(ctx) + c := &checkMarch{ + ctx: ctx, + tokens: make(chan struct{}, ci.Checkers), + opt: *opt, + } + lastErr := ListFn(ctx, opt.Fdst, func(obj fs.Object) { + c.checkSum(ctx, obj, download, hashes, hashType) + }) + c.wg.Wait() // wait for background go-routines + + // make census of unhandled sums + fi := filter.GetConfig(ctx) + for filename, hash := range hashes { + if hash == "" { // the sum has been successfully consumed + continue + } + if !fi.IncludeRemote(filename) { // the file was filtered out + continue + } + // filesystem missed the file, sum wasn't consumed + err := fmt.Errorf("file not in %v", opt.Fdst) + fs.Errorf(filename, "%v", err) + _ = fs.CountError(ctx, err) + if lastErr == nil { + lastErr = err + } + c.dstFilesMissing.Add(1) + c.reportFilename(filename, opt.MissingOnDst, '+') + } + + return c.reportResults(ctx, lastErr) +} + +// checkSum checks single object against golden hashes +func (c *checkMarch) checkSum(ctx context.Context, obj fs.Object, download bool, hashes HashSums, hashType hash.Type) { + normalizedRemote := ApplyTransforms(ctx, obj.Remote()) + c.ioMu.Lock() + sumHash, sumFound := hashes[normalizedRemote] + hashes[normalizedRemote] = "" // mark sum as consumed + c.ioMu.Unlock() + + if !sumFound && c.opt.OneWay { + return + } + + var err error + tr := accounting.Stats(ctx).NewCheckingTransfer(obj, "hashing") + defer tr.Done(ctx, err) + + if !sumFound { + err = errors.New("sum not found") + _ = fs.CountError(ctx, err) + fs.Errorf(obj, "%v", err) + c.differences.Add(1) + c.srcFilesMissing.Add(1) + c.report(obj, c.opt.MissingOnSrc, '-') + return + } + + if !download { + var objHash string + objHash, err = obj.Hash(ctx, hashType) + c.matchSum(ctx, sumHash, objHash, obj, err, hashType) + return + } + + c.wg.Add(1) + c.tokens <- struct{}{} // put a token to limit concurrency + go func() { + var ( + objHash string + err error + in io.ReadCloser + ) + defer func() { + c.matchSum(ctx, sumHash, objHash, obj, err, hashType) + <-c.tokens // get the token back to free up a slot + c.wg.Done() + }() + if in, err = Open(ctx, obj); err != nil { + return + } + tr := accounting.Stats(ctx).NewTransfer(obj, nil) + in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer + defer func() { + tr.Done(ctx, nil) // will close the stream + }() + hashVals, err2 := hash.StreamTypes(in, hash.NewHashSet(hashType)) + if err2 != nil { + err = err2 // pass to matchSum + return + } + objHash = hashVals[hashType] + }() +} + +// matchSum sums up the results of hashsum matching for an object +func (c *checkMarch) matchSum(ctx context.Context, sumHash, objHash string, obj fs.Object, err error, hashType hash.Type) { + switch { + case err != nil: + _ = fs.CountError(ctx, err) + fs.Errorf(obj, "Failed to calculate hash: %v", err) + c.report(obj, c.opt.Error, '!') + case sumHash == "": + err = errors.New("duplicate file") + _ = fs.CountError(ctx, err) + fs.Errorf(obj, "%v", err) + c.report(obj, c.opt.Error, '!') + case objHash == "": + fs.Debugf(nil, "%v = %s (sum)", hashType, sumHash) + fs.Debugf(obj, "%v - could not check hash (%v)", hashType, c.opt.Fdst) + c.noHashes.Add(1) + c.matches.Add(1) + c.report(obj, c.opt.Match, '=') + case objHash == sumHash: + fs.Debugf(obj, "%v = %s OK", hashType, sumHash) + c.matches.Add(1) + c.report(obj, c.opt.Match, '=') + default: + err = errors.New("files differ") + _ = fs.CountError(ctx, err) + fs.Debugf(nil, "%v = %s (sum)", hashType, sumHash) + fs.Debugf(obj, "%v = %s (%v)", hashType, objHash, c.opt.Fdst) + fs.Errorf(obj, "%v", err) + c.differences.Add(1) + c.report(obj, c.opt.Differ, '*') + } +} + +// HashSums represents a parsed SUM file +type HashSums map[string]string + +// ParseSumFile parses a hash SUM file and returns hashes as a map +func ParseSumFile(ctx context.Context, sumFile fs.Object) (HashSums, error) { + rd, err := Open(ctx, sumFile) + if err != nil { + return nil, err + } + parser := bufio.NewReader(rd) + + const maxWarn = 3 + numWarn := 0 + + re := regexp.MustCompile(`^([^ ]+) [ *](.+)$`) + hashes := HashSums{} + for lineNo := 0; true; lineNo++ { + lineBytes, _, err := parser.ReadLine() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + line := string(lineBytes) + if line == "" { + continue + } + + fields := re.FindStringSubmatch(ApplyTransforms(ctx, line)) + if fields == nil { + numWarn++ + if numWarn <= maxWarn { + fs.Logf(sumFile, "improperly formatted checksum line %d", lineNo) + } + continue + } + + sum, file := fields[1], fields[2] + if hashes[file] != "" { + numWarn++ + if numWarn <= maxWarn { + fs.Logf(sumFile, "duplicate file on checksum line %d", lineNo) + } + continue + } + + // We've standardised on lower case checksums in rclone internals. + hashes[file] = strings.ToLower(sum) + } + + if numWarn > maxWarn { + fs.Logf(sumFile, "%d warning(s) suppressed...", numWarn-maxWarn) + } + if err = rd.Close(); err != nil { + return nil, err + } + return hashes, nil +} diff --git a/fs/operations/check_test.go b/fs/operations/check_test.go new file mode 100644 index 0000000..e6c3092 --- /dev/null +++ b/fs/operations/check_test.go @@ -0,0 +1,617 @@ +package operations_test + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "sort" + "strings" + "testing" + + "github.com/rclone/rclone/cmd/bisync/bilib" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/lib/readers" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/text/unicode/norm" +) + +func testCheck(t *testing.T, checkFunction func(ctx context.Context, opt *operations.CheckOpt) error) { + r := fstest.NewRun(t) + ctx := context.Background() + ci := fs.GetConfig(ctx) + + addBuffers := func(opt *operations.CheckOpt) { + opt.Combined = new(bytes.Buffer) + opt.MissingOnSrc = new(bytes.Buffer) + opt.MissingOnDst = new(bytes.Buffer) + opt.Match = new(bytes.Buffer) + opt.Differ = new(bytes.Buffer) + opt.Error = new(bytes.Buffer) + } + + sortLines := func(in string) []string { + if in == "" { + return []string{} + } + lines := strings.Split(in, "\n") + sort.Strings(lines) + return lines + } + + checkBuffer := func(name string, want map[string]string, out io.Writer) { + expected := want[name] + buf, ok := out.(*bytes.Buffer) + require.True(t, ok) + assert.Equal(t, sortLines(expected), sortLines(buf.String()), name) + } + + checkBuffers := func(opt *operations.CheckOpt, want map[string]string) { + checkBuffer("combined", want, opt.Combined) + checkBuffer("missingonsrc", want, opt.MissingOnSrc) + checkBuffer("missingondst", want, opt.MissingOnDst) + checkBuffer("match", want, opt.Match) + checkBuffer("differ", want, opt.Differ) + checkBuffer("error", want, opt.Error) + } + + check := func(i int, wantErrors int64, wantChecks int64, oneway bool, wantOutput map[string]string) { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + accounting.GlobalStats().ResetCounters() + opt := operations.CheckOpt{ + Fdst: r.Fremote, + Fsrc: r.Flocal, + OneWay: oneway, + } + addBuffers(&opt) + var err error + buf := bilib.CaptureOutput(func() { + err = checkFunction(ctx, &opt) + }) + gotErrors := accounting.GlobalStats().GetErrors() + gotChecks := accounting.GlobalStats().GetChecks() + if wantErrors == 0 && err != nil { + t.Errorf("%d: Got error when not expecting one: %v", i, err) + } + if wantErrors != 0 && err == nil { + t.Errorf("%d: No error when expecting one", i) + } + if wantErrors != gotErrors { + t.Errorf("%d: Expecting %d errors but got %d", i, wantErrors, gotErrors) + } + if gotChecks > 0 && !strings.Contains(string(buf), "matching files") { + t.Errorf("%d: Total files matching line missing", i) + } + if wantChecks != gotChecks { + t.Errorf("%d: Expecting %d total matching files but got %d", i, wantChecks, gotChecks) + } + checkBuffers(&opt, wantOutput) + }) + } + + file1 := r.WriteBoth(ctx, "rutabaga", "is tasty", t3) + r.CheckRemoteItems(t, file1) + r.CheckLocalItems(t, file1) + check(1, 0, 1, false, map[string]string{ + "combined": "= rutabaga\n", + "missingonsrc": "", + "missingondst": "", + "match": "rutabaga\n", + "differ": "", + "error": "", + }) + + file2 := r.WriteFile("potato2", "------------------------------------------------------------", t1) + r.CheckLocalItems(t, file1, file2) + check(2, 1, 1, false, map[string]string{ + "combined": "+ potato2\n= rutabaga\n", + "missingonsrc": "", + "missingondst": "potato2\n", + "match": "rutabaga\n", + "differ": "", + "error": "", + }) + + file3 := r.WriteObject(ctx, "empty space", "-", t2) + r.CheckRemoteItems(t, file1, file3) + check(3, 2, 1, false, map[string]string{ + "combined": "- empty space\n+ potato2\n= rutabaga\n", + "missingonsrc": "empty space\n", + "missingondst": "potato2\n", + "match": "rutabaga\n", + "differ": "", + "error": "", + }) + + file2r := file2 + if ci.SizeOnly { + file2r = r.WriteObject(ctx, "potato2", "--Some-Differences-But-Size-Only-Is-Enabled-----------------", t1) + } else { + r.WriteObject(ctx, "potato2", "------------------------------------------------------------", t1) + } + r.CheckRemoteItems(t, file1, file2r, file3) + check(4, 1, 2, false, map[string]string{ + "combined": "- empty space\n= potato2\n= rutabaga\n", + "missingonsrc": "empty space\n", + "missingondst": "", + "match": "rutabaga\npotato2\n", + "differ": "", + "error": "", + }) + + file3r := file3 + file3l := r.WriteFile("empty space", "DIFFER", t2) + r.CheckLocalItems(t, file1, file2, file3l) + check(5, 1, 3, false, map[string]string{ + "combined": "* empty space\n= potato2\n= rutabaga\n", + "missingonsrc": "", + "missingondst": "", + "match": "potato2\nrutabaga\n", + "differ": "empty space\n", + "error": "", + }) + + file4 := r.WriteObject(ctx, "remotepotato", "------------------------------------------------------------", t1) + r.CheckRemoteItems(t, file1, file2r, file3r, file4) + check(6, 2, 3, false, map[string]string{ + "combined": "* empty space\n= potato2\n= rutabaga\n- remotepotato\n", + "missingonsrc": "remotepotato\n", + "missingondst": "", + "match": "potato2\nrutabaga\n", + "differ": "empty space\n", + "error": "", + }) + check(7, 1, 3, true, map[string]string{ + "combined": "* empty space\n= potato2\n= rutabaga\n", + "missingonsrc": "", + "missingondst": "", + "match": "potato2\nrutabaga\n", + "differ": "empty space\n", + "error": "", + }) +} + +func TestCheck(t *testing.T) { + testCheck(t, operations.Check) +} + +func TestCheckFsError(t *testing.T) { + ctx := context.Background() + dstFs, err := fs.NewFs(ctx, "nonexistent") + if err != nil { + t.Fatal(err) + } + srcFs, err := fs.NewFs(ctx, "nonexistent") + if err != nil { + t.Fatal(err) + } + opt := operations.CheckOpt{ + Fdst: dstFs, + Fsrc: srcFs, + OneWay: false, + } + err = operations.Check(ctx, &opt) + require.Error(t, err) +} + +func TestCheckDownload(t *testing.T) { + testCheck(t, operations.CheckDownload) +} + +func TestCheckSizeOnly(t *testing.T) { + ctx := context.Background() + ci := fs.GetConfig(ctx) + ci.SizeOnly = true + defer func() { ci.SizeOnly = false }() + TestCheck(t) +} + +func TestCheckEqualReaders(t *testing.T) { + b65a := make([]byte, 65*1024) + b65b := make([]byte, 65*1024) + b65b[len(b65b)-1] = 1 + b66 := make([]byte, 66*1024) + + equal, err := operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b65a)) + assert.NoError(t, err) + assert.Equal(t, equal, true) + + equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b65b)) + assert.NoError(t, err) + assert.Equal(t, equal, false) + + equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), bytes.NewBuffer(b66)) + assert.NoError(t, err) + assert.Equal(t, equal, false) + + equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b66), bytes.NewBuffer(b65a)) + assert.NoError(t, err) + assert.Equal(t, equal, false) + + myErr := errors.New("sentinel") + wrap := func(b []byte) io.Reader { + r := bytes.NewBuffer(b) + e := readers.ErrorReader{Err: myErr} + return io.MultiReader(r, e) + } + + equal, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b65a)) + assert.Equal(t, myErr, err) + assert.Equal(t, equal, false) + + equal, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b65b)) + assert.Equal(t, myErr, err) + assert.Equal(t, equal, false) + + equal, err = operations.CheckEqualReaders(wrap(b65a), bytes.NewBuffer(b66)) + assert.Equal(t, myErr, err) + assert.Equal(t, equal, false) + + equal, err = operations.CheckEqualReaders(wrap(b66), bytes.NewBuffer(b65a)) + assert.Equal(t, myErr, err) + assert.Equal(t, equal, false) + + equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b65a)) + assert.Equal(t, myErr, err) + assert.Equal(t, equal, false) + + equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b65b)) + assert.Equal(t, myErr, err) + assert.Equal(t, equal, false) + + equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b65a), wrap(b66)) + assert.Equal(t, myErr, err) + assert.Equal(t, equal, false) + + equal, err = operations.CheckEqualReaders(bytes.NewBuffer(b66), wrap(b65a)) + assert.Equal(t, myErr, err) + assert.Equal(t, equal, false) +} + +func TestParseSumFile(t *testing.T) { + r := fstest.NewRun(t) + ctx := context.Background() + + const sumFile = "test.sum" + + samples := []struct { + hash, sep, name string + ok bool + }{ + {"1", " ", "file1", true}, + {"2", " *", "file2", true}, + {"3", " ", " file3 ", true}, + {"4", " ", "\tfile3\t", true}, + {"5", " ", "file5", false}, + {"6", "\t", "file6", false}, + {"7", " \t", " file7 ", false}, + {"", " ", "file8", false}, + {"", "", "file9", false}, + } + + for _, eol := range []string{"\n", "\r\n"} { + data := &bytes.Buffer{} + wantNum := 0 + for _, s := range samples { + _, _ = data.WriteString(s.hash + s.sep + s.name + eol) + if s.ok { + wantNum++ + } + } + + _ = r.WriteObject(ctx, sumFile, data.String(), t1) + file := fstest.NewObject(ctx, t, r.Fremote, sumFile) + sums, err := operations.ParseSumFile(ctx, file) + assert.NoError(t, err) + + assert.Equal(t, wantNum, len(sums)) + for _, s := range samples { + if s.ok { + assert.Equal(t, s.hash, sums[s.name]) + } + } + } +} + +func testCheckSum(t *testing.T, download bool) { + const dataDir = "data" + const sumFile = "test.sum" + + hashType := hash.MD5 + const ( + testString1 = "Hello, World!" + testDigest1 = "65a8e27d8879283831b664bd8b7f0ad4" + testDigest1Upper = "65A8E27D8879283831B664BD8B7F0AD4" + testString2 = "I am the walrus" + testDigest2 = "87396e030ef3f5b35bbf85c0a09a4fb3" + testDigest2Mixed = "87396e030EF3f5b35BBf85c0a09a4FB3" + ) + + type wantType map[string]string + + ctx := context.Background() + r := fstest.NewRun(t) + + subRemote := r.FremoteName + if !strings.HasSuffix(subRemote, ":") { + subRemote += "/" + } + subRemote += dataDir + dataFs, err := fs.NewFs(ctx, subRemote) + require.NoError(t, err) + + if !download && !dataFs.Hashes().Contains(hashType) { + t.Skipf("%s lacks %s, skipping", dataFs, hashType) + } + + makeFile := func(name, content string) fstest.Item { + remote := dataDir + "/" + name + return r.WriteObject(ctx, remote, content, t1) + } + + makeSums := func(sums operations.HashSums) fstest.Item { + files := make([]string, 0, len(sums)) + for name := range sums { + files = append(files, name) + } + sort.Strings(files) + buf := &bytes.Buffer{} + for _, name := range files { + _, _ = fmt.Fprintf(buf, "%s %s\n", sums[name], name) + } + return r.WriteObject(ctx, sumFile, buf.String(), t1) + } + + sortLines := func(in string) []string { + if in == "" { + return []string{} + } + lines := strings.Split(in, "\n") + sort.Strings(lines) + return lines + } + + checkResult := func(runNo int, want wantType, name string, out io.Writer) { + expected := want[name] + buf, ok := out.(*bytes.Buffer) + require.True(t, ok) + assert.Equal(t, sortLines(expected), sortLines(buf.String()), "wrong %s result in run %d", name, runNo) + } + + checkRun := func(runNo, wantChecks, wantErrors int, want wantType) { + accounting.GlobalStats().ResetCounters() + + opt := operations.CheckOpt{ + Combined: new(bytes.Buffer), + Match: new(bytes.Buffer), + Differ: new(bytes.Buffer), + Error: new(bytes.Buffer), + MissingOnSrc: new(bytes.Buffer), + MissingOnDst: new(bytes.Buffer), + } + var err error + buf := bilib.CaptureOutput(func() { + err = operations.CheckSum(ctx, dataFs, r.Fremote, sumFile, hashType, &opt, download) + }) + gotErrors := int(accounting.GlobalStats().GetErrors()) + if wantErrors == 0 { + assert.NoError(t, err, "unexpected error in run %d", runNo) + } + if wantErrors > 0 { + assert.Error(t, err, "no expected error in run %d", runNo) + } + assert.Equal(t, wantErrors, gotErrors, "wrong error count in run %d", runNo) + + gotChecks := int(accounting.GlobalStats().GetChecks()) + if wantChecks > 0 || gotChecks > 0 { + assert.Contains(t, string(buf), "matching files", "missing matching files in run %d", runNo) + } + assert.Equal(t, wantChecks, gotChecks, "wrong number of checks in run %d", runNo) + + checkResult(runNo, want, "combined", opt.Combined) + checkResult(runNo, want, "missingonsrc", opt.MissingOnSrc) + checkResult(runNo, want, "missingondst", opt.MissingOnDst) + checkResult(runNo, want, "match", opt.Match) + checkResult(runNo, want, "differ", opt.Differ) + checkResult(runNo, want, "error", opt.Error) + } + + check := func(runNo, wantChecks, wantErrors int, wantResults wantType) { + runName := fmt.Sprintf("subtest%d", runNo) + t.Run(runName, func(t *testing.T) { + checkRun(runNo, wantChecks, wantErrors, wantResults) + }) + } + + file1 := makeFile("banana", testString1) + fcsums := makeSums(operations.HashSums{ + "banana": testDigest1, + }) + r.CheckRemoteItems(t, fcsums, file1) + check(1, 1, 0, wantType{ + "combined": "= banana\n", + "missingonsrc": "", + "missingondst": "", + "match": "banana\n", + "differ": "", + "error": "", + }) + + file2 := makeFile("potato", testString2) + fcsums = makeSums(operations.HashSums{ + "banana": testDigest1, + }) + r.CheckRemoteItems(t, fcsums, file1, file2) + check(2, 2, 1, wantType{ + "combined": "- potato\n= banana\n", + "missingonsrc": "potato\n", + "missingondst": "", + "match": "banana\n", + "differ": "", + "error": "", + }) + + fcsums = makeSums(operations.HashSums{ + "banana": testDigest1, + "potato": testDigest2, + }) + r.CheckRemoteItems(t, fcsums, file1, file2) + check(3, 2, 0, wantType{ + "combined": "= potato\n= banana\n", + "missingonsrc": "", + "missingondst": "", + "match": "banana\npotato\n", + "differ": "", + "error": "", + }) + + fcsums = makeSums(operations.HashSums{ + "banana": testDigest2, + "potato": testDigest2, + }) + r.CheckRemoteItems(t, fcsums, file1, file2) + check(4, 2, 1, wantType{ + "combined": "* banana\n= potato\n", + "missingonsrc": "", + "missingondst": "", + "match": "potato\n", + "differ": "banana\n", + "error": "", + }) + + fcsums = makeSums(operations.HashSums{ + "banana": testDigest1, + "potato": testDigest2, + "orange": testDigest2, + }) + r.CheckRemoteItems(t, fcsums, file1, file2) + check(5, 2, 1, wantType{ + "combined": "+ orange\n= potato\n= banana\n", + "missingonsrc": "", + "missingondst": "orange\n", + "match": "banana\npotato\n", + "differ": "", + "error": "", + }) + + fcsums = makeSums(operations.HashSums{ + "banana": testDigest1, + "potato": testDigest1, + "orange": testDigest2, + }) + r.CheckRemoteItems(t, fcsums, file1, file2) + check(6, 2, 2, wantType{ + "combined": "+ orange\n* potato\n= banana\n", + "missingonsrc": "", + "missingondst": "orange\n", + "match": "banana\n", + "differ": "potato\n", + "error": "", + }) + + // test mixed-case checksums + file1 = makeFile("banana", testString1) + file2 = makeFile("potato", testString2) + fcsums = makeSums(operations.HashSums{ + "banana": testDigest1Upper, + "potato": testDigest2Mixed, + }) + r.CheckRemoteItems(t, fcsums, file1, file2) + check(7, 2, 0, wantType{ + "combined": "= banana\n= potato\n", + "missingonsrc": "", + "missingondst": "", + "match": "banana\npotato\n", + "differ": "", + "error": "", + }) +} + +func TestCheckSum(t *testing.T) { + testCheckSum(t, false) +} + +func TestCheckSumDownload(t *testing.T) { + testCheckSum(t, true) +} + +func TestApplyTransforms(t *testing.T) { + var ( + hashType = hash.MD5 + content = "Hello, World!" + hash = "65a8e27d8879283831b664bd8b7f0ad4" + nfc = norm.NFC.String(norm.NFD.String("測試_Русский___ě_áñ")) + nfd = norm.NFD.String(nfc) + nfcx2 = nfc + nfc + nfdx2 = nfd + nfd + both = nfc + nfd + upper = "HELLO, WORLD!" + lower = "hello, world!" + upperlowermixed = "HeLlO, wOrLd!" + ) + + testScenario := func(checkfileName, remotefileName, scenario string) { + r := fstest.NewRunIndividual(t) + ctx := context.Background() + ci := fs.GetConfig(ctx) + opt := operations.CheckOpt{} + + remotefile := r.WriteObject(ctx, remotefileName, content, t2) + // test whether remote is capable of running test + entries, err := r.Fremote.List(ctx, "") + assert.NoError(t, err) + if entries.Len() == 1 && entries[0].Remote() != remotefileName { + t.Skipf("Fs is incapable of running test, skipping: %s (expected: %s (%s) actual: %s (%s))", scenario, remotefileName, detectEncoding(remotefileName), entries[0].Remote(), detectEncoding(entries[0].Remote())) + } + + checkfile := r.WriteFile("test.sum", hash+" "+checkfileName, t2) + r.CheckLocalItems(t, checkfile) + assert.False(t, checkfileName == remotefile.Path, "Values match but should not: %s %s", checkfileName, remotefile.Path) + + testname := scenario + " (without normalization)" + println(testname) + ci.NoUnicodeNormalization = true + ci.IgnoreCaseSync = false + accounting.GlobalStats().ResetCounters() + err = operations.CheckSum(ctx, r.Fremote, r.Flocal, "test.sum", hashType, &opt, true) + assert.Error(t, err, "no expected error for %s %v %v", testname, checkfileName, remotefileName) + + testname = scenario + " (with normalization)" + println(testname) + ci.NoUnicodeNormalization = false + ci.IgnoreCaseSync = true + accounting.GlobalStats().ResetCounters() + err = operations.CheckSum(ctx, r.Fremote, r.Flocal, "test.sum", hashType, &opt, true) + assert.NoError(t, err, "unexpected error for %s %v %v", testname, checkfileName, remotefileName) + } + + testScenario(upper, lower, "upper checkfile vs. lower remote") + testScenario(lower, upper, "lower checkfile vs. upper remote") + testScenario(lower, upperlowermixed, "lower checkfile vs. upperlowermixed remote") + testScenario(upperlowermixed, upper, "upperlowermixed checkfile vs. upper remote") + testScenario(nfd, nfc, "NFD checkfile vs. NFC remote") + testScenario(nfc, nfd, "NFC checkfile vs. NFD remote") + testScenario(nfdx2, both, "NFDx2 checkfile vs. both remote") + testScenario(nfcx2, both, "NFCx2 checkfile vs. both remote") + testScenario(both, nfdx2, "both checkfile vs. NFDx2 remote") + testScenario(both, nfcx2, "both checkfile vs. NFCx2 remote") +} + +func detectEncoding(s string) string { + if norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) { + return "BOTH" + } + if !norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) { + return "NFD" + } + if norm.NFC.IsNormalString(s) && !norm.NFD.IsNormalString(s) { + return "NFC" + } + return "OTHER" +} diff --git a/fs/operations/copy.go b/fs/operations/copy.go new file mode 100644 index 0000000..f7871d2 --- /dev/null +++ b/fs/operations/copy.go @@ -0,0 +1,422 @@ +// This file implements operations.Copy +// +// This is probably the most important operation in rclone. + +package operations + +import ( + "context" + "errors" + "fmt" + "hash/crc32" + "io" + "path" + "strings" + "time" + "unicode/utf8" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/atexit" + "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/transform" +) + +// State of the copy +type copy struct { + f fs.Fs // destination fs.Fs + dstFeatures *fs.Features // Features() for fs.Fs + dst fs.Object // destination object to update, may be nil + remote string // destination path, used if dst is nil + src fs.Object // source object + ci *fs.ConfigInfo // current config + maxTries int // max number of tries to do the copy + doUpdate bool // whether we are updating an existing file or not + hashType hash.Type // common hash to use + hashOption *fs.HashesOption // open option for the common hash + tr *accounting.Transfer // accounting for the transfer + inplace bool // set if we are updating inplace and not using a partial name + remoteForCopy string // the name used for the transfer, either remote or remote+".partial" +} + +// Used to remove a failed copy +func (c *copy) removeFailedCopy(ctx context.Context, o fs.Object) { + if o == nil { + return + } + fs.Infof(o, "Removing failed copy") + err := o.Remove(ctx) + if err != nil { + fs.Infof(o, "Failed to remove failed copy: %s", err) + } +} + +// Used to remove a failed partial copy +func (c *copy) removeFailedPartialCopy(ctx context.Context, f fs.Fs, remote string) { + o, err := f.NewObject(ctx, remote) + if errors.Is(err, fs.ErrorObjectNotFound) { + // Assume object has been deleted + return + } + if err != nil { + fs.Infof(remote, "Failed to remove failed partial copy: %s", err) + return + } + c.removeFailedCopy(ctx, o) +} + +// TruncateString s to n bytes. +// +// If s is valid UTF-8 then this may truncate to fewer than n bytes to +// make the returned string also valid UTF-8. +func TruncateString(s string, n int) string { + truncated := s[:n] + if !utf8.ValidString(s) { + // If input string wasn't valid UTF-8 then just return the truncation + return truncated + } + for len(truncated) > 0 { + if utf8.ValidString(truncated) { + return truncated + } + // Remove 1 byte until valid + truncated = truncated[:len(truncated)-1] + } + return truncated +} + +// Check to see if we should be using a partial name and return the name for the copy and the inplace flag +func (c *copy) checkPartial(ctx context.Context) (remoteForCopy string, inplace bool, err error) { + remoteForCopy = c.remote + if c.ci.Inplace || c.dstFeatures.Move == nil || !c.dstFeatures.PartialUploads || strings.HasSuffix(c.remote, ".rclonelink") { + return remoteForCopy, true, nil + } + if len(c.ci.PartialSuffix) > 16 { + return remoteForCopy, true, fmt.Errorf("expecting length of --partial-suffix to be not greater than %d but got %d", 16, len(c.ci.PartialSuffix)) + } + // Avoid making the leaf name longer if it's already lengthy to avoid + // trouble with file name length limits. + + // generate a stable random suffix by hashing the filename and fingerprint + hasher := crc32.New(crc32.IEEETable) + _, _ = hasher.Write([]byte(c.remote)) + _, _ = hasher.Write([]byte(fs.Fingerprint(ctx, c.src, true))) + hash := hasher.Sum32() + + suffix := fmt.Sprintf(".%08x%s", hash, c.ci.PartialSuffix) + base := path.Base(remoteForCopy) + if len(base) > 100 { + remoteForCopy = TruncateString(remoteForCopy, len(remoteForCopy)-len(suffix)) + suffix + } else { + remoteForCopy += suffix + } + return remoteForCopy, false, nil +} + +// Check to see if we have hit max transfer limits +func (c *copy) checkLimits(ctx context.Context) (err error) { + if c.ci.MaxTransfer < 0 { + return nil + } + var bytesSoFar int64 + if c.ci.CutoffMode == fs.CutoffModeCautious { + bytesSoFar = accounting.Stats(ctx).GetBytesWithPending() + c.src.Size() + } else { + bytesSoFar = accounting.Stats(ctx).GetBytes() + } + if bytesSoFar >= int64(c.ci.MaxTransfer) { + if c.ci.CutoffMode == fs.CutoffModeHard { + return accounting.ErrorMaxTransferLimitReachedFatal + } + return accounting.ErrorMaxTransferLimitReachedGraceful + } + return nil +} + +// Server side copy c.src to (c.f, c.remoteForCopy) if possible or return fs.ErrorCantCopy if not +func (c *copy) serverSideCopy(ctx context.Context) (actionTaken string, newDst fs.Object, err error) { + doCopy := c.dstFeatures.Copy + serverSideCopyOK := false + if doCopy == nil { + serverSideCopyOK = false + } else if SameConfig(c.src.Fs(), c.f) { + serverSideCopyOK = true + } else if SameRemoteType(c.src.Fs(), c.f) { + serverSideCopyOK = c.dstFeatures.ServerSideAcrossConfigs || c.ci.ServerSideAcrossConfigs + } + if !serverSideCopyOK { + return actionTaken, nil, fs.ErrorCantCopy + } + in := c.tr.Account(ctx, nil) // account the transfer + in.ServerSideTransferStart() + newDst, err = doCopy(ctx, c.src, c.remoteForCopy) + if err == nil { + in.ServerSideCopyEnd(newDst.Size()) // account the bytes for the server-side transfer + } + _ = in.Close() + if errors.Is(err, fs.ErrorCantCopy) { + c.tr.Reset(ctx) // skip incomplete accounting - will be overwritten by the manual copy + } + actionTaken = "Copied (server-side copy)" + return actionTaken, newDst, err +} + +// Copy c.src to (c.f, c.remoteForCopy) using multiThreadCopy +func (c *copy) multiThreadCopy(ctx context.Context, uploadOptions []fs.OpenOption) (actionTaken string, newDst fs.Object, err error) { + newDst, err = multiThreadCopy(ctx, c.f, c.remoteForCopy, c.src, c.ci.MultiThreadStreams, c.tr, uploadOptions...) + if c.doUpdate { + actionTaken = "Multi-thread Copied (replaced existing)" + } else { + actionTaken = "Multi-thread Copied (new)" + } + return actionTaken, newDst, err +} + +// Copy the stream from in to (c.f, c.remoteForCopy) and close it +// +// Use Rcat to handle both remotes supporting and not supporting PutStream. +func (c *copy) rcat(ctx context.Context, in io.ReadCloser) (actionTaken string, newDst fs.Object, err error) { + // Make any metadata to pass to rcat + var meta fs.Metadata + if c.ci.Metadata { + meta, err = fs.GetMetadata(ctx, c.src) + if err != nil { + fs.Errorf(c.src, "Failed to read metadata: %v", err) + } + } + + // NB Rcat closes in0 + fsrc, ok := c.src.Fs().(fs.Fs) + if !ok { + fsrc = nil + } + newDst, err = rcatSrc(ctx, c.f, c.remoteForCopy, in, c.src.ModTime(ctx), meta, fsrc) + if c.doUpdate { + actionTaken = "Copied (Rcat, replaced existing)" + } else { + actionTaken = "Copied (Rcat, new)" + } + return actionTaken, newDst, err +} + +// Copy the stream from in to (c.f, c.remoteForCopy) and close it +func (c *copy) updateOrPut(ctx context.Context, in io.ReadCloser, uploadOptions []fs.OpenOption) (actionTaken string, newDst fs.Object, err error) { + // account and buffer the transfer + inAcc := c.tr.Account(ctx, in).WithBuffer() + var wrappedSrc fs.ObjectInfo = c.src + + // We try to pass the original object if possible + if c.src.Remote() != c.remoteForCopy { + wrappedSrc = fs.NewOverrideRemote(c.src, c.remoteForCopy) + } + if c.doUpdate && c.inplace { + err = c.dst.Update(ctx, inAcc, wrappedSrc, uploadOptions...) + // Make sure newDst is c.dst since we updated it + if err == nil { + newDst = c.dst + } + } else { + newDst, err = c.f.Put(ctx, inAcc, wrappedSrc, uploadOptions...) + } + closeErr := inAcc.Close() + if err == nil { + err = closeErr + } + if c.doUpdate { + actionTaken = "Copied (replaced existing)" + } else { + actionTaken = "Copied (new)" + } + return actionTaken, newDst, err +} + +// Do a manual copy by reading the bytes and writing them +func (c *copy) manualCopy(ctx context.Context) (actionTaken string, newDst fs.Object, err error) { + // Remove partial files on premature exit + if !c.inplace { + defer atexit.Unregister(atexit.Register(func() { + ctx := context.Background() + c.removeFailedPartialCopy(ctx, c.f, c.remoteForCopy) + })) + } + + // Options for the upload + uploadOptions := []fs.OpenOption{c.hashOption} + for _, option := range c.ci.UploadHeaders { + uploadOptions = append(uploadOptions, option) + } + if c.ci.MetadataSet != nil { + uploadOptions = append(uploadOptions, fs.MetadataOption(c.ci.MetadataSet)) + } + + // Options for the download + downloadOptions := []fs.OpenOption{c.hashOption} + for _, option := range c.ci.DownloadHeaders { + downloadOptions = append(downloadOptions, option) + } + + if doMultiThreadCopy(ctx, c.f, c.src) { + return c.multiThreadCopy(ctx, uploadOptions) + } + + var in io.ReadCloser + in, err = Open(ctx, c.src, downloadOptions...) + if err != nil { + return actionTaken, nil, fmt.Errorf("failed to open source object: %w", err) + } + + // Note that c.rcat and c.updateOrPut close in + if c.src.Size() == -1 { + return c.rcat(ctx, in) + } + return c.updateOrPut(ctx, in, uploadOptions) +} + +// Verify the copy +func (c *copy) verify(ctx context.Context, newDst fs.Object) (err error) { + // Verify sizes are the same after transfer + if sizeDiffers(ctx, c.src, newDst) { + return fmt.Errorf("corrupted on transfer: sizes differ src(%s) %d vs dst(%s) %d", c.src.Fs(), c.src.Size(), newDst.Fs(), newDst.Size()) + } + // Verify hashes are the same after transfer - ignoring blank hashes + if c.hashType != hash.None { + // checkHashes has logs and counts errors + equal, _, srcSum, dstSum, _ := checkHashes(ctx, c.src, newDst, c.hashType) + if !equal { + return fmt.Errorf("corrupted on transfer: %v hashes differ src(%s) %q vs dst(%s) %q", c.hashType, c.src.Fs(), srcSum, newDst.Fs(), dstSum) + } + } + return nil +} + +// copy src object to dst or f if nil. If dst is nil then it uses +// remote as the name of the new object. +// +// It returns the destination object if possible. Note that this may +// be nil. +func (c *copy) copy(ctx context.Context) (newDst fs.Object, err error) { + var actionTaken string + retry := true + for tries := 0; retry && tries < c.maxTries; tries++ { + // Check we haven't hit any accounting limits + err = c.checkLimits(ctx) + if err != nil { + return nil, err + } + + // Try server side copy + actionTaken, newDst, err = c.serverSideCopy(ctx) + + // If can't server-side copy, do it manually + if errors.Is(err, fs.ErrorCantCopy) { + actionTaken, newDst, err = c.manualCopy(ctx) + } + + // End if ctx is in error + if fserrors.ContextError(ctx, &err) { + break + } + + // Retry if err returned a retry error + retry = false + if fserrors.IsRetryError(err) || fserrors.ShouldRetry(err) { + retry = true + } else if t, ok := pacer.IsRetryAfter(err); ok { + fs.Debugf(c.src, "Sleeping for %v (as indicated by the server) to obey Retry-After error: %v", t, err) + time.Sleep(t) + retry = true + } + if retry { + fs.Debugf(c.src, "Received error: %v - low level retry %d/%d", err, tries, c.maxTries) + c.tr.Reset(ctx) // skip incomplete accounting - will be overwritten by retry + continue + } + } + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(c.src, "Failed to copy: %v", err) + if !c.inplace { + c.removeFailedPartialCopy(ctx, c.f, c.remoteForCopy) + } + return newDst, err + } + + // Verify the copy + err = c.verify(ctx, newDst) + if err != nil { + fs.Errorf(newDst, "%v", err) + err = fs.CountError(ctx, err) + c.removeFailedCopy(ctx, newDst) + return nil, err + } + + // Move the copied file to its real destination. + if !c.inplace && c.remoteForCopy != c.remote { + movedNewDst, err := c.dstFeatures.Move(ctx, newDst, c.remote) + if err != nil { + fs.Errorf(newDst, "partial file rename failed: %v", err) + err = fs.CountError(ctx, err) + c.removeFailedCopy(ctx, newDst) + return nil, err + } + fs.Debugf(newDst, "renamed to: %s", c.remote) + newDst = movedNewDst + } + + // Log what we have done + if newDst != nil && c.src.String() != newDst.String() { + actionTaken = fmt.Sprintf("%s to: %s", actionTaken, newDst.String()) + } + fs.Infof(c.src, "%s%s", actionTaken, fs.LogValueHide("size", fs.SizeSuffix(c.src.Size()))) + + return newDst, nil +} + +// Copy src object to dst or f if nil. If dst is nil then it uses +// remote as the name of the new object. +// +// It returns the destination object if possible. Note that this may +// be nil. +func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) { + ci := fs.GetConfig(ctx) + tr := accounting.Stats(ctx).NewTransfer(src, f) + defer func() { + tr.Done(ctx, err) + }() + if SkipDestructive(ctx, src, "copy") { + in := tr.Account(ctx, nil) + in.DryRun(src.Size()) + return newDst, nil + } + c := ©{ + f: f, + dstFeatures: f.Features(), + dst: dst, + remote: transform.Path(ctx, remote, false), + src: src, + ci: ci, + tr: tr, + maxTries: ci.LowLevelRetries, + doUpdate: dst != nil, + } + c.hashType, c.hashOption = CommonHash(ctx, f, src.Fs()) + if c.dst != nil { + c.remote = transform.Path(ctx, c.dst.Remote(), false) + } + // Are we using partials? + // + // If so set the flag and update the name we use for the copy + c.remoteForCopy, c.inplace, err = c.checkPartial(ctx) + if err != nil { + return nil, err + } + // Do the copy now everything is set up + return c.copy(ctx) +} + +// CopyFile moves a single file possibly to a new name +func CopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) { + return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, true, false) +} diff --git a/fs/operations/copy_test.go b/fs/operations/copy_test.go new file mode 100644 index 0000000..e717f3b --- /dev/null +++ b/fs/operations/copy_test.go @@ -0,0 +1,534 @@ +package operations_test + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "os" + "path" + "runtime" + "sort" + "strings" + "testing" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/sync" + "github.com/rclone/rclone/fstest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTruncateString(t *testing.T) { + for _, test := range []struct { + in string + n int + want string + }{ + { + in: "", + n: 0, + want: "", + }, { + in: "Hello World", + n: 5, + want: "Hello", + }, { + in: "ááááá", + n: 5, + want: "áá", + }, { + in: "ááááá\xFF\xFF", + n: 5, + want: "áá\xc3", + }, { + in: "世世世世世", + n: 7, + want: "世世", + }, { + in: "🙂🙂🙂🙂🙂", + n: 16, + want: "🙂🙂🙂🙂", + }, { + in: "🙂🙂🙂🙂🙂", + n: 15, + want: "🙂🙂🙂", + }, { + in: "🙂🙂🙂🙂🙂", + n: 14, + want: "🙂🙂🙂", + }, { + in: "🙂🙂🙂🙂🙂", + n: 13, + want: "🙂🙂🙂", + }, { + in: "🙂🙂🙂🙂🙂", + n: 12, + want: "🙂🙂🙂", + }, { + in: "🙂🙂🙂🙂🙂", + n: 11, + want: "🙂🙂", + }, { + in: "𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ", + n: 100, + want: "𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢ", + }, { + in: "a𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ", + n: 100, + want: "a𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢ", + }, { + in: "aa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ", + n: 100, + want: "aa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱ", + }, { + in: "aaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ", + n: 100, + want: "aaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱ", + }, { + in: "aaaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽⁱˢⁱᵒⁿᵃʳʸ", + n: 100, + want: "aaaa𝓝𝓸𝓫𝓸𝓭𝔂 𝓲𝓼 𝓱𝓸𝓶𝓮 ᴬ ⱽⁱˢⁱᵗ ᶠʳᵒᵐ ᵗʰᵉ ⱽ", + }, + } { + got := operations.TruncateString(test.in, test.n) + assert.Equal(t, test.want, got, fmt.Sprintf("In %q", test.in)) + assert.LessOrEqual(t, len(got), test.n) + assert.GreaterOrEqual(t, len(got), test.n-3) + } +} + +func TestCopyFile(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + + file1 := r.WriteFile("file1", "file1 contents", t1) + r.CheckLocalItems(t, file1) + + file2 := file1 + file2.Path = "sub/file2" + + err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) + + err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) + + err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path) + require.NoError(t, err) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) +} + +// Find the longest file name for writing to local +func maxLengthFileName(t *testing.T, r *fstest.Run) string { + require.NoError(t, r.Flocal.Mkdir(context.Background(), "")) // create the root + const maxLen = 16 * 1024 + name := strings.Repeat("A", maxLen) + i := sort.Search(len(name), func(i int) (fail bool) { + filePath := path.Join(r.LocalName, name[:i]) + err := os.WriteFile(filePath, []byte{0}, 0777) + if err != nil { + return true + } + err = os.Remove(filePath) + if err != nil { + t.Logf("Failed to remove test file: %v", err) + } + return false + }) + return name[:i-1] +} + +// Check we can copy a file of maximum name length +func TestCopyLongFile(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + if !r.Fremote.Features().IsLocal { + t.Skip("Test only runs on local") + } + + // Find the maximum length of file we can write + name := maxLengthFileName(t, r) + t.Logf("Max length of file name is %d", len(name)) + file1 := r.WriteFile(name, "file1 contents", t1) + r.CheckLocalItems(t, file1) + + err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) +} + +func TestCopyFileBackupDir(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + if !operations.CanServerSideMove(r.Fremote) { + t.Skip("Skipping test as remote does not support server-side move or copy") + } + + ci.BackupDir = r.FremoteName + "/backup" + + file1 := r.WriteFile("dst/file1", "file1 contents", t1) + r.CheckLocalItems(t, file1) + + file1old := r.WriteObject(ctx, "dst/file1", "file1 contents old", t1) + r.CheckRemoteItems(t, file1old) + + err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t, file1) + file1old.Path = "backup/dst/file1" + r.CheckRemoteItems(t, file1old, file1) +} + +// Test with CompareDest set +func TestCopyFileCompareDest(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + ci.CompareDest = []string{r.FremoteName + "/CompareDest"} + fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") + require.NoError(t, err) + + // check empty dest, empty compare + file1 := r.WriteFile("one", "one", t1) + r.CheckLocalItems(t, file1) + + err = operations.CopyFile(ctx, fdst, r.Flocal, file1.Path, file1.Path) + require.NoError(t, err) + + file1dst := file1 + file1dst.Path = "dst/one" + + r.CheckRemoteItems(t, file1dst) + + // check old dest, empty compare + file1b := r.WriteFile("one", "onet2", t2) + r.CheckRemoteItems(t, file1dst) + r.CheckLocalItems(t, file1b) + + err = operations.CopyFile(ctx, fdst, r.Flocal, file1b.Path, file1b.Path) + require.NoError(t, err) + + file1bdst := file1b + file1bdst.Path = "dst/one" + + r.CheckRemoteItems(t, file1bdst) + + // check old dest, new compare + file3 := r.WriteObject(ctx, "dst/one", "one", t1) + file2 := r.WriteObject(ctx, "CompareDest/one", "onet2", t2) + file1c := r.WriteFile("one", "onet2", t2) + r.CheckRemoteItems(t, file2, file3) + r.CheckLocalItems(t, file1c) + + err = operations.CopyFile(ctx, fdst, r.Flocal, file1c.Path, file1c.Path) + require.NoError(t, err) + + r.CheckRemoteItems(t, file2, file3) + + // check empty dest, new compare + file4 := r.WriteObject(ctx, "CompareDest/two", "two", t2) + file5 := r.WriteFile("two", "two", t2) + r.CheckRemoteItems(t, file2, file3, file4) + r.CheckLocalItems(t, file1c, file5) + + err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path) + require.NoError(t, err) + + r.CheckRemoteItems(t, file2, file3, file4) + + // check new dest, new compare + err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path) + require.NoError(t, err) + + r.CheckRemoteItems(t, file2, file3, file4) + + // check empty dest, old compare + file5b := r.WriteFile("two", "twot3", t3) + r.CheckRemoteItems(t, file2, file3, file4) + r.CheckLocalItems(t, file1c, file5b) + + err = operations.CopyFile(ctx, fdst, r.Flocal, file5b.Path, file5b.Path) + require.NoError(t, err) + + file5bdst := file5b + file5bdst.Path = "dst/two" + + r.CheckRemoteItems(t, file2, file3, file4, file5bdst) +} + +// Test with CopyDest set +func TestCopyFileCopyDest(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + if r.Fremote.Features().Copy == nil { + t.Skip("Skipping test as remote does not support server-side copy") + } + + ci.CopyDest = []string{r.FremoteName + "/CopyDest"} + + fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") + require.NoError(t, err) + + // check empty dest, empty copy + file1 := r.WriteFile("one", "one", t1) + r.CheckLocalItems(t, file1) + + err = operations.CopyFile(ctx, fdst, r.Flocal, file1.Path, file1.Path) + require.NoError(t, err) + + file1dst := file1 + file1dst.Path = "dst/one" + + r.CheckRemoteItems(t, file1dst) + + // check old dest, empty copy + file1b := r.WriteFile("one", "onet2", t2) + r.CheckRemoteItems(t, file1dst) + r.CheckLocalItems(t, file1b) + + err = operations.CopyFile(ctx, fdst, r.Flocal, file1b.Path, file1b.Path) + require.NoError(t, err) + + file1bdst := file1b + file1bdst.Path = "dst/one" + + r.CheckRemoteItems(t, file1bdst) + + // check old dest, new copy, backup-dir + + ci.BackupDir = r.FremoteName + "/BackupDir" + + file3 := r.WriteObject(ctx, "dst/one", "one", t1) + file2 := r.WriteObject(ctx, "CopyDest/one", "onet2", t2) + file1c := r.WriteFile("one", "onet2", t2) + r.CheckRemoteItems(t, file2, file3) + r.CheckLocalItems(t, file1c) + + err = operations.CopyFile(ctx, fdst, r.Flocal, file1c.Path, file1c.Path) + require.NoError(t, err) + + file2dst := file2 + file2dst.Path = "dst/one" + file3.Path = "BackupDir/one" + + r.CheckRemoteItems(t, file2, file2dst, file3) + ci.BackupDir = "" + + // check empty dest, new copy + file4 := r.WriteObject(ctx, "CopyDest/two", "two", t2) + file5 := r.WriteFile("two", "two", t2) + r.CheckRemoteItems(t, file2, file2dst, file3, file4) + r.CheckLocalItems(t, file1c, file5) + + err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path) + require.NoError(t, err) + + file4dst := file4 + file4dst.Path = "dst/two" + + r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst) + + // check new dest, new copy + err = operations.CopyFile(ctx, fdst, r.Flocal, file5.Path, file5.Path) + require.NoError(t, err) + + r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst) + + // check empty dest, old copy + file6 := r.WriteObject(ctx, "CopyDest/three", "three", t2) + file7 := r.WriteFile("three", "threet3", t3) + r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6) + r.CheckLocalItems(t, file1c, file5, file7) + + err = operations.CopyFile(ctx, fdst, r.Flocal, file7.Path, file7.Path) + require.NoError(t, err) + + file7dst := file7 + file7dst.Path = "dst/three" + + r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6, file7dst) +} + +func TestCopyInplace(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + if !r.Fremote.Features().PartialUploads { + t.Skip("Partial uploads not supported") + } + + ci.Inplace = true + + file1 := r.WriteFile("file1", "file1 contents", t1) + r.CheckLocalItems(t, file1) + + file2 := file1 + file2.Path = "sub/file2" + + err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) + + err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) + + err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path) + require.NoError(t, err) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) +} + +func TestCopyLongFileName(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + if !r.Fremote.Features().PartialUploads { + t.Skip("Partial uploads not supported") + } + + ci.Inplace = false // the default + + file1 := r.WriteFile("file1", "file1 contents", t1) + r.CheckLocalItems(t, file1) + + file2 := file1 + file2.Path = "sub/" + strings.Repeat("file2", 30) + + err := operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) + + err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) + + err = operations.CopyFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path) + require.NoError(t, err) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) +} + +func TestCopyLongFileNameCollision(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + if !r.Fremote.Features().PartialUploads { + t.Skip("Partial uploads not supported") + } + + ci.Inplace = false + ci.Transfers = 4 + + // Write a lot of identical files with long names + files := make([]fstest.Item, 10) + namePrefix := strings.Repeat("file1", 30) + for i := range files { + files[i] = r.WriteFile(fmt.Sprintf("%s%02d", namePrefix, i), "file1 contents", t1) + } + r.CheckLocalItems(t, files...) + + err := sync.CopyDir(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + r.CheckLocalItems(t, files...) + r.CheckRemoteItems(t, files...) +} + +func TestCopyFileMaxTransfer(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + defer accounting.Stats(ctx).ResetCounters() + + const sizeCutoff = 2048 + + // Make random incompressible data + randomData := make([]byte, sizeCutoff) + _, err := rand.Read(randomData) + require.NoError(t, err) + randomString := string(randomData) + + file1 := r.WriteFile("TestCopyFileMaxTransfer/file1", "file1 contents", t1) + file2 := r.WriteFile("TestCopyFileMaxTransfer/file2", "file2 contents"+randomString, t2) + file3 := r.WriteFile("TestCopyFileMaxTransfer/file3", "file3 contents"+randomString, t2) + file4 := r.WriteFile("TestCopyFileMaxTransfer/file4", "file4 contents"+randomString, t2) + + // Cutoff mode: Hard + ci.MaxTransfer = sizeCutoff + ci.CutoffMode = fs.CutoffModeHard + + if runtime.GOOS == "darwin" { + // disable server-side copies as they don't count towards transfer size stats + r.Flocal.Features().Disable("Copy") + if r.Fremote.Features().IsLocal { + r.Fremote.Features().Disable("Copy") + } + } + + // file1: Show a small file gets transferred OK + accounting.Stats(ctx).ResetCounters() + err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t, file1, file2, file3, file4) + r.CheckRemoteItems(t, file1) + + // file2: show a large file does not get transferred + accounting.Stats(ctx).ResetCounters() + err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file2.Path) + require.NotNil(t, err, "Did not get expected max transfer limit error") + if !errors.Is(err, accounting.ErrorMaxTransferLimitReachedFatal) { + t.Log("Expecting error to contain accounting.ErrorMaxTransferLimitReachedFatal") + // Sometimes the backends or their SDKs don't pass the + // error through properly, so check that it at least + // has the text we expect in. + assert.Contains(t, err.Error(), "max transfer limit reached") + } + r.CheckLocalItems(t, file1, file2, file3, file4) + r.CheckRemoteItems(t, file1) + + // Cutoff mode: Cautious + ci.CutoffMode = fs.CutoffModeCautious + + // file3: show a large file does not get transferred + accounting.Stats(ctx).ResetCounters() + err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file3.Path, file3.Path) + require.NotNil(t, err) + assert.True(t, errors.Is(err, accounting.ErrorMaxTransferLimitReachedGraceful)) + r.CheckLocalItems(t, file1, file2, file3, file4) + r.CheckRemoteItems(t, file1) + + if isChunker(r.Fremote) { + t.Log("skipping remainder of test for chunker as it involves multiple transfers") + return + } + + // Cutoff mode: Soft + ci.CutoffMode = fs.CutoffModeSoft + + // file4: show a large file does get transferred this time + accounting.Stats(ctx).ResetCounters() + err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file4.Path, file4.Path) + require.NoError(t, err) + r.CheckLocalItems(t, file1, file2, file3, file4) + r.CheckRemoteItems(t, file1, file4) +} diff --git a/fs/operations/dedupe.go b/fs/operations/dedupe.go new file mode 100644 index 0000000..60c66c4 --- /dev/null +++ b/fs/operations/dedupe.go @@ -0,0 +1,506 @@ +// dedupe - gets rid of identical files remotes which can have duplicate file names (drive, mega) + +package operations + +import ( + "context" + "fmt" + "path" + "sort" + "strings" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/walk" +) + +// dedupeRename renames the objs slice to different names +func dedupeRename(ctx context.Context, f fs.Fs, remote string, objs []fs.Object) { + doMove := f.Features().Move + if doMove == nil { + fs.Fatalf(nil, "Fs %v doesn't support Move", f) + } + ext := path.Ext(remote) + base := remote[:len(remote)-len(ext)] + +outer: + for i, o := range objs { + suffix := 1 + newName := fmt.Sprintf("%s-%d%s", base, i+suffix, ext) + _, err := f.NewObject(ctx, newName) + for ; err != fs.ErrorObjectNotFound; suffix++ { + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(o, "Failed to check for existing object: %v", err) + continue outer + } + if suffix > 100 { + fs.Errorf(o, "Could not find an available new name") + continue outer + } + newName = fmt.Sprintf("%s-%d%s", base, i+suffix, ext) + _, err = f.NewObject(ctx, newName) + } + if !SkipDestructive(ctx, o, "rename") { + newObj, err := doMove(ctx, o, newName) + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(o, "Failed to rename: %v", err) + continue + } + fs.Infof(newObj, "renamed from: %v", o) + } + } +} + +// dedupeDeleteAllButOne deletes all but the one in keep +func dedupeDeleteAllButOne(ctx context.Context, keep int, remote string, objs []fs.Object) { + count := 0 + for i, o := range objs { + if i == keep { + continue + } + err := DeleteFile(ctx, o) + if err == nil { + count++ + } + } + if count > 0 { + fs.Logf(remote, "Deleted %d extra copies", count) + } +} + +// dedupeDeleteIdentical deletes all but one of identical (by hash) copies +func dedupeDeleteIdentical(ctx context.Context, ht hash.Type, remote string, objs []fs.Object) (remainingObjs []fs.Object) { + ci := fs.GetConfig(ctx) + + // Make map of IDs + IDs := make(map[string]int, len(objs)) + for _, o := range objs { + if do, ok := o.(fs.IDer); ok { + if ID := do.ID(); ID != "" { + IDs[ID]++ + } + } + } + + // Remove duplicate IDs + newObjs := objs[:0] + for _, o := range objs { + if do, ok := o.(fs.IDer); ok { + if ID := do.ID(); ID != "" { + if IDs[ID] <= 1 { + newObjs = append(newObjs, o) + } else { + fs.Logf(o, "Ignoring as it appears %d times in the listing and deleting would lead to data loss", IDs[ID]) + } + } + } + } + objs = newObjs + + // See how many of these duplicates are identical + dupesByID := make(map[string][]fs.Object, len(objs)) + for _, o := range objs { + ID := "" + if ci.SizeOnly && o.Size() >= 0 { + ID = fmt.Sprintf("size %d", o.Size()) + } else if ht != hash.None { + hashValue, err := o.Hash(ctx, ht) + if err == nil && hashValue != "" { + ID = fmt.Sprintf("%v %s", ht, hashValue) + } + } + if ID == "" { + remainingObjs = append(remainingObjs, o) + } else { + dupesByID[ID] = append(dupesByID[ID], o) + } + } + + // Delete identical duplicates, filling remainingObjs with the ones remaining + for ID, dupes := range dupesByID { + remainingObjs = append(remainingObjs, dupes[0]) + if len(dupes) > 1 { + fs.Logf(remote, "Deleting %d/%d identical duplicates (%s)", len(dupes)-1, len(dupes), ID) + for _, o := range dupes[1:] { + err := DeleteFile(ctx, o) + if err != nil { + remainingObjs = append(remainingObjs, o) + } + } + } + } + + return remainingObjs +} + +// dedupeList lists the duplicates and does nothing +func dedupeList(ctx context.Context, f fs.Fs, ht hash.Type, remote string, objs []fs.Object, byHash bool) { + fmt.Printf("%s: %d duplicates\n", remote, len(objs)) + for i, o := range objs { + hashValue := "" + if ht != hash.None { + var err error + hashValue, err = o.Hash(ctx, ht) + if err != nil { + hashValue = err.Error() + } + } + if byHash { + fmt.Printf(" %d: %12d bytes, %s, %s\n", i+1, o.Size(), o.ModTime(ctx).Local().Format("2006-01-02 15:04:05.000000000"), o.Remote()) + } else { + fmt.Printf(" %d: %12d bytes, %s, %v %32s\n", i+1, o.Size(), o.ModTime(ctx).Local().Format("2006-01-02 15:04:05.000000000"), ht, hashValue) + } + } +} + +// dedupeInteractive interactively dedupes the slice of objects +func dedupeInteractive(ctx context.Context, f fs.Fs, ht hash.Type, remote string, objs []fs.Object, byHash bool) bool { + dedupeList(ctx, f, ht, remote, objs, byHash) + commands := []string{"sSkip and do nothing", "kKeep just one (choose which in next step)"} + if !byHash { + commands = append(commands, "rRename all to be different (by changing file.jpg to file-1.jpg)") + } + commands = append(commands, "qQuit") + switch config.Command(commands) { + case 's': + case 'k': + keep := config.ChooseNumber("Enter the number of the file to keep", 1, len(objs)) + dedupeDeleteAllButOne(ctx, keep-1, remote, objs) + case 'r': + dedupeRename(ctx, f, remote, objs) + case 'q': + return false + } + return true +} + +// DeduplicateMode is how the dedupe command chooses what to do +type DeduplicateMode int + +// Deduplicate modes +const ( + DeduplicateInteractive DeduplicateMode = iota // interactively ask the user + DeduplicateSkip // skip all conflicts + DeduplicateFirst // choose the first object + DeduplicateNewest // choose the newest object + DeduplicateOldest // choose the oldest object + DeduplicateRename // rename the objects + DeduplicateLargest // choose the largest object + DeduplicateSmallest // choose the smallest object + DeduplicateList // list duplicates only +) + +func (x DeduplicateMode) String() string { + switch x { + case DeduplicateInteractive: + return "interactive" + case DeduplicateSkip: + return "skip" + case DeduplicateFirst: + return "first" + case DeduplicateNewest: + return "newest" + case DeduplicateOldest: + return "oldest" + case DeduplicateRename: + return "rename" + case DeduplicateLargest: + return "largest" + case DeduplicateSmallest: + return "smallest" + case DeduplicateList: + return "list" + } + return "unknown" +} + +// Set a DeduplicateMode from a string +func (x *DeduplicateMode) Set(s string) error { + switch strings.ToLower(s) { + case "interactive": + *x = DeduplicateInteractive + case "skip": + *x = DeduplicateSkip + case "first": + *x = DeduplicateFirst + case "newest": + *x = DeduplicateNewest + case "oldest": + *x = DeduplicateOldest + case "rename": + *x = DeduplicateRename + case "largest": + *x = DeduplicateLargest + case "smallest": + *x = DeduplicateSmallest + case "list": + *x = DeduplicateList + default: + return fmt.Errorf("unknown mode for dedupe %q", s) + } + return nil +} + +// Type of the value +func (x *DeduplicateMode) Type() string { + return "string" +} + +// Directory with entry count and links to parents +type dedupeDir struct { + dir fs.Directory + parent string + count int +} + +// Map of directories by ID with recursive counts +type dedupeDirsMap map[string]*dedupeDir + +func (dm dedupeDirsMap) get(id string) *dedupeDir { + d := dm[id] + if d == nil { + d = &dedupeDir{} + dm[id] = d + } + return d +} + +func (dm dedupeDirsMap) increment(parent string) { + if parent != "" { + d := dm.get(parent) + d.count++ + dm.increment(d.parent) + } +} + +// dedupeFindDuplicateDirs scans f for duplicate directories +func dedupeFindDuplicateDirs(ctx context.Context, f fs.Fs) (duplicateDirs [][]*dedupeDir, err error) { + dirsByID := dedupeDirsMap{} + dirs := map[string][]*dedupeDir{} + + ci := fs.GetConfig(ctx) + err = walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListAll, func(entries fs.DirEntries) error { + for _, entry := range entries { + tr := accounting.Stats(ctx).NewCheckingTransfer(entry, "merging") + + remote := entry.Remote() + parentRemote := path.Dir(remote) + if parentRemote == "." { + parentRemote = "" + } + + // Obtain ID of the object parent, if known. + // (This usually means that backend allows duplicate paths) + // Fall back to remote parent path, if unavailable. + var parent string + if entryParentIDer, ok := entry.(fs.ParentIDer); ok { + parent = entryParentIDer.ParentID() + } + if parent == "" { + parent = parentRemote + } + + var ID string + if entryIDer, ok := entry.(fs.IDer); ok { + ID = entryIDer.ID() + } + if ID == "" { + ID = remote + } + + if fsDir, ok := entry.(fs.Directory); ok { + d := dirsByID.get(ID) + d.dir = fsDir + d.parent = parent + dirs[remote] = append(dirs[remote], d) + } + + dirsByID.increment(parent) + tr.Done(ctx, nil) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("find duplicate dirs: %w", err) + } + + // Make sure parents are before children + duplicateNames := []string{} + for name, ds := range dirs { + if len(ds) > 1 { + duplicateNames = append(duplicateNames, name) + } + } + sort.Strings(duplicateNames) + for _, name := range duplicateNames { + duplicateDirs = append(duplicateDirs, dirs[name]) + } + + return +} + +// dedupeMergeDuplicateDirs merges all the duplicate directories found +func dedupeMergeDuplicateDirs(ctx context.Context, f fs.Fs, duplicateDirs [][]*dedupeDir) error { + mergeDirs := f.Features().MergeDirs + if mergeDirs == nil { + return fmt.Errorf("%v: can't merge directories", f) + } + dirCacheFlush := f.Features().DirCacheFlush + if dirCacheFlush == nil { + return fmt.Errorf("%v: can't flush dir cache", f) + } + for _, dedupeDirs := range duplicateDirs { + if SkipDestructive(ctx, dedupeDirs[0].dir, "merge duplicate directories") { + continue + } + + // Put largest directory in front to minimize movements + fsDirs := []fs.Directory{} + largestCount := -1 + largestIdx := 0 + for i, d := range dedupeDirs { + fsDirs = append(fsDirs, d.dir) + if d.count > largestCount { + largestIdx = i + largestCount = d.count + } + } + fsDirs[largestIdx], fsDirs[0] = fsDirs[0], fsDirs[largestIdx] + + fs.Infof(fsDirs[0], "Merging contents of duplicate directories") + err := mergeDirs(ctx, fsDirs) + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(nil, "merge duplicate dirs: %v", err) + } + } + dirCacheFlush() + return nil +} + +// sort oldest first +func sortOldestFirst(objs []fs.Object) { + sort.Slice(objs, func(i, j int) bool { + return objs[i].ModTime(context.TODO()).Before(objs[j].ModTime(context.TODO())) + }) +} + +// sort smallest first +func sortSmallestFirst(objs []fs.Object) { + sort.Slice(objs, func(i, j int) bool { + return objs[i].Size() < objs[j].Size() + }) +} + +// Deduplicate interactively finds duplicate files and offers to +// delete all but one or rename them to be different. Only useful with +// Google Drive which can have duplicate file names. +func Deduplicate(ctx context.Context, f fs.Fs, mode DeduplicateMode, byHash bool) error { + ci := fs.GetConfig(ctx) + // find a hash to use + ht := f.Hashes().GetOne() + what := "names" + if byHash { + if ht == hash.None { + return fmt.Errorf("%v has no hashes", f) + } + what = ht.String() + " hashes" + } + fs.Infof(f, "Looking for duplicate %s using %v mode.", what, mode) + + // Find duplicate directories first and fix them + if !byHash { + duplicateDirs, err := dedupeFindDuplicateDirs(ctx, f) + if err != nil { + return err + } + if len(duplicateDirs) > 0 { + if mode != DeduplicateList { + err = dedupeMergeDuplicateDirs(ctx, f, duplicateDirs) + if err != nil { + return err + } + } else { + for _, dedupeDirs := range duplicateDirs { + remote := dedupeDirs[0].dir.Remote() + fmt.Printf("%s: %d duplicates of this directory\n", remote, len(dedupeDirs)) + } + } + } + } + + // Now find duplicate files + files := map[string][]fs.Object{} + err := walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error { + entries.ForObject(func(o fs.Object) { + tr := accounting.Stats(ctx).NewCheckingTransfer(o, "checking") + defer tr.Done(ctx, nil) + + var remote string + var err error + if byHash { + remote, err = o.Hash(ctx, ht) + if err != nil { + fs.Errorf(o, "Failed to hash: %v", err) + remote = "" + } + } else { + remote = o.Remote() + } + if remote != "" { + files[remote] = append(files[remote], o) + } + }) + return nil + }) + if err != nil { + return err + } + + for remote, objs := range files { + if len(objs) <= 1 { + continue + } + fs.Logf(remote, "Found %d files with duplicate %s", len(objs), what) + if !byHash && mode != DeduplicateList { + objs = dedupeDeleteIdentical(ctx, ht, remote, objs) + if len(objs) <= 1 { + fs.Logf(remote, "All duplicates removed") + continue + } + } + switch mode { + case DeduplicateInteractive: + if !dedupeInteractive(ctx, f, ht, remote, objs, byHash) { + return nil + } + case DeduplicateFirst: + dedupeDeleteAllButOne(ctx, 0, remote, objs) + case DeduplicateNewest: + sortOldestFirst(objs) + dedupeDeleteAllButOne(ctx, len(objs)-1, remote, objs) + case DeduplicateOldest: + sortOldestFirst(objs) + dedupeDeleteAllButOne(ctx, 0, remote, objs) + case DeduplicateRename: + dedupeRename(ctx, f, remote, objs) + case DeduplicateLargest: + sortSmallestFirst(objs) + dedupeDeleteAllButOne(ctx, len(objs)-1, remote, objs) + case DeduplicateSmallest: + sortSmallestFirst(objs) + dedupeDeleteAllButOne(ctx, 0, remote, objs) + case DeduplicateSkip: + fs.Logf(remote, "Skipping %d files with duplicate %s", len(objs), what) + case DeduplicateList: + dedupeList(ctx, f, ht, remote, objs, byHash) + default: + //skip + } + } + return nil +} diff --git a/fs/operations/dedupe_test.go b/fs/operations/dedupe_test.go new file mode 100644 index 0000000..15a4e2d --- /dev/null +++ b/fs/operations/dedupe_test.go @@ -0,0 +1,280 @@ +package operations_test + +import ( + "context" + "testing" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/lib/random" + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Check flag satisfies the interface +var _ pflag.Value = (*operations.DeduplicateMode)(nil) + +func skipIfCantDedupe(t *testing.T, f fs.Fs) { + if !f.Features().DuplicateFiles { + t.Skip("Can't test deduplicate - no duplicate files possible") + } + if f.Features().PutUnchecked == nil { + t.Skip("Can't test deduplicate - no PutUnchecked") + } + if f.Features().MergeDirs == nil { + t.Skip("Can't test deduplicate - no MergeDirs") + } +} + +func skipIfNoHash(t *testing.T, f fs.Fs) { + if f.Hashes().GetOne() == hash.None { + t.Skip("Can't run this test without a hash") + } +} + +func skipIfNoModTime(t *testing.T, f fs.Fs) { + if f.Precision() >= fs.ModTimeNotSupported { + t.Skip("Can't run this test without modtimes") + } +} + +func TestDeduplicateInteractive(t *testing.T) { + r := fstest.NewRun(t) + skipIfCantDedupe(t, r.Fremote) + skipIfNoHash(t, r.Fremote) + + file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1) + file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1) + file3 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1) + r.CheckWithDuplicates(t, file1, file2, file3) + + err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateInteractive, false) + require.NoError(t, err) + + r.CheckRemoteItems(t, file1) +} + +func TestDeduplicateSkip(t *testing.T) { + r := fstest.NewRun(t) + skipIfCantDedupe(t, r.Fremote) + haveHash := r.Fremote.Hashes().GetOne() != hash.None + + file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1) + files := []fstest.Item{file1} + if haveHash { + file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1) + files = append(files, file2) + } + file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t1) + files = append(files, file3) + r.CheckWithDuplicates(t, files...) + + err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateSkip, false) + require.NoError(t, err) + + r.CheckWithDuplicates(t, file1, file3) +} + +func TestDeduplicateSizeOnly(t *testing.T) { + r := fstest.NewRun(t) + skipIfCantDedupe(t, r.Fremote) + ctx := context.Background() + ci := fs.GetConfig(ctx) + + file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1) + file2 := r.WriteUncheckedObject(context.Background(), "one", "THIS IS ONE", t1) + file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t1) + r.CheckWithDuplicates(t, file1, file2, file3) + + ci.SizeOnly = true + defer func() { + ci.SizeOnly = false + }() + + err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateSkip, false) + require.NoError(t, err) + + r.CheckWithDuplicates(t, file1, file3) +} + +func TestDeduplicateFirst(t *testing.T) { + r := fstest.NewRun(t) + skipIfCantDedupe(t, r.Fremote) + + file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1) + file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one A", t1) + file3 := r.WriteUncheckedObject(context.Background(), "one", "This is one BB", t1) + r.CheckWithDuplicates(t, file1, file2, file3) + + err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateFirst, false) + require.NoError(t, err) + + // list until we get one object + var objects, size int64 + for try := 1; try <= *fstest.ListRetries; try++ { + objects, size, _, err = operations.Count(context.Background(), r.Fremote) + require.NoError(t, err) + if objects == 1 { + break + } + time.Sleep(time.Second) + } + assert.Equal(t, int64(1), objects) + if size != file1.Size && size != file2.Size && size != file3.Size { + t.Errorf("Size not one of the object sizes %d", size) + } +} + +func TestDeduplicateNewest(t *testing.T) { + r := fstest.NewRun(t) + skipIfCantDedupe(t, r.Fremote) + skipIfNoModTime(t, r.Fremote) + + file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1) + file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2) + file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3) + r.CheckWithDuplicates(t, file1, file2, file3) + + err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateNewest, false) + require.NoError(t, err) + + r.CheckRemoteItems(t, file3) +} + +func TestDeduplicateNewestByHash(t *testing.T) { + r := fstest.NewRun(t) + skipIfNoHash(t, r.Fremote) + skipIfNoModTime(t, r.Fremote) + contents := random.String(100) + + file1 := r.WriteObject(context.Background(), "one", contents, t1) + file2 := r.WriteObject(context.Background(), "also/one", contents, t2) + file3 := r.WriteObject(context.Background(), "another", contents, t3) + file4 := r.WriteObject(context.Background(), "not-one", "stuff", t3) + r.CheckRemoteItems(t, file1, file2, file3, file4) + + err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateNewest, true) + require.NoError(t, err) + + r.CheckRemoteItems(t, file3, file4) +} + +func TestDeduplicateOldest(t *testing.T) { + r := fstest.NewRun(t) + skipIfCantDedupe(t, r.Fremote) + + file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1) + file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2) + file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3) + r.CheckWithDuplicates(t, file1, file2, file3) + + err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateOldest, false) + require.NoError(t, err) + + r.CheckRemoteItems(t, file1) +} + +func TestDeduplicateLargest(t *testing.T) { + r := fstest.NewRun(t) + skipIfCantDedupe(t, r.Fremote) + + file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1) + file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2) + file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3) + r.CheckWithDuplicates(t, file1, file2, file3) + + err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateLargest, false) + require.NoError(t, err) + + r.CheckRemoteItems(t, file3) +} + +func TestDeduplicateSmallest(t *testing.T) { + r := fstest.NewRun(t) + skipIfCantDedupe(t, r.Fremote) + + file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1) + file2 := r.WriteUncheckedObject(context.Background(), "one", "This is one too", t2) + file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t3) + r.CheckWithDuplicates(t, file1, file2, file3) + + err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateSmallest, false) + require.NoError(t, err) + + r.CheckRemoteItems(t, file1) +} + +func TestDeduplicateRename(t *testing.T) { + r := fstest.NewRun(t) + skipIfCantDedupe(t, r.Fremote) + + file1 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is one", t1) + file2 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is one too", t2) + file3 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is another one", t3) + file4 := r.WriteUncheckedObject(context.Background(), "one-1.txt", "This is not a duplicate", t1) + r.CheckWithDuplicates(t, file1, file2, file3, file4) + + err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateRename, false) + require.NoError(t, err) + + require.NoError(t, walk.ListR(context.Background(), r.Fremote, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error { + entries.ForObject(func(o fs.Object) { + remote := o.Remote() + if remote != "one-1.txt" && + remote != "one-2.txt" && + remote != "one-3.txt" && + remote != "one-4.txt" { + t.Errorf("Bad file name after rename %q", remote) + } + size := o.Size() + if size != file1.Size && + size != file2.Size && + size != file3.Size && + size != file4.Size { + t.Errorf("Size not one of the object sizes %d", size) + } + if remote == "one-1.txt" && size != file4.Size { + t.Errorf("Existing non-duplicate file modified %q", remote) + } + }) + return nil + })) +} + +// This should really be a unit test, but the test framework there +// doesn't have enough tools to make it easy +func TestMergeDirs(t *testing.T) { + r := fstest.NewRun(t) + + mergeDirs := r.Fremote.Features().MergeDirs + if mergeDirs == nil { + t.Skip("Can't merge directories") + } + + file1 := r.WriteObject(context.Background(), "dupe1/one.txt", "This is one", t1) + file2 := r.WriteObject(context.Background(), "dupe2/two.txt", "This is one too", t2) + file3 := r.WriteObject(context.Background(), "dupe3/three.txt", "This is another one", t3) + + objs, dirs, err := walk.GetAll(context.Background(), r.Fremote, "", true, 1) + require.NoError(t, err) + assert.Equal(t, 3, len(dirs)) + assert.Equal(t, 0, len(objs)) + + err = mergeDirs(context.Background(), dirs) + require.NoError(t, err) + + file2.Path = "dupe1/two.txt" + file3.Path = "dupe1/three.txt" + r.CheckRemoteItems(t, file1, file2, file3) + + objs, dirs, err = walk.GetAll(context.Background(), r.Fremote, "", true, 1) + require.NoError(t, err) + assert.Equal(t, 1, len(dirs)) + assert.Equal(t, 0, len(objs)) + assert.Equal(t, "dupe1", dirs[0].Remote()) +} diff --git a/fs/operations/listdirsorted_test.go b/fs/operations/listdirsorted_test.go new file mode 100644 index 0000000..4719d6c --- /dev/null +++ b/fs/operations/listdirsorted_test.go @@ -0,0 +1,126 @@ +package operations_test + +import ( + "context" + "testing" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/list" + "github.com/rclone/rclone/fstest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testListDirSorted is integration testing code in fs/list/list.go +// which can't be tested there due to import loops. +func testListDirSorted(t *testing.T, listFn func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error)) { + r := fstest.NewRun(t) + + ctx := context.Background() + fi := filter.GetConfig(ctx) + fi.Opt.MaxSize = 10 + defer func() { + fi.Opt.MaxSize = -1 + }() + + files := []fstest.Item{ + r.WriteObject(context.Background(), "a.txt", "hello world", t1), + r.WriteObject(context.Background(), "zend.txt", "hello", t1), + r.WriteObject(context.Background(), "sub dir/hello world", "hello world", t1), + r.WriteObject(context.Background(), "sub dir/hello world2", "hello world", t1), + r.WriteObject(context.Background(), "sub dir/ignore dir/.ignore", "-", t1), + r.WriteObject(context.Background(), "sub dir/ignore dir/should be ignored", "to ignore", t1), + r.WriteObject(context.Background(), "sub dir/sub sub dir/hello world3", "hello world", t1), + } + r.CheckRemoteItems(t, files...) + var items fs.DirEntries + var err error + + // Turn the DirEntry into a name, ending with a / if it is a + // dir + str := func(i int) string { + item := items[i] + name := item.Remote() + switch item.(type) { + case fs.Object: + case fs.Directory: + name += "/" + default: + t.Fatalf("Unknown type %+v", item) + } + return name + } + + items, err = listFn(context.Background(), r.Fremote, true, "") + require.NoError(t, err) + require.Len(t, items, 3) + assert.Equal(t, "a.txt", str(0)) + assert.Equal(t, "sub dir/", str(1)) + assert.Equal(t, "zend.txt", str(2)) + + items, err = listFn(context.Background(), r.Fremote, false, "") + require.NoError(t, err) + require.Len(t, items, 2) + assert.Equal(t, "sub dir/", str(0)) + assert.Equal(t, "zend.txt", str(1)) + + items, err = listFn(context.Background(), r.Fremote, true, "sub dir") + require.NoError(t, err) + require.Len(t, items, 4) + assert.Equal(t, "sub dir/hello world", str(0)) + assert.Equal(t, "sub dir/hello world2", str(1)) + assert.Equal(t, "sub dir/ignore dir/", str(2)) + assert.Equal(t, "sub dir/sub sub dir/", str(3)) + + items, err = listFn(context.Background(), r.Fremote, false, "sub dir") + require.NoError(t, err) + require.Len(t, items, 2) + assert.Equal(t, "sub dir/ignore dir/", str(0)) + assert.Equal(t, "sub dir/sub sub dir/", str(1)) + + // testing ignore file + fi.Opt.ExcludeFile = []string{".ignore"} + + items, err = listFn(context.Background(), r.Fremote, false, "sub dir") + require.NoError(t, err) + require.Len(t, items, 1) + assert.Equal(t, "sub dir/sub sub dir/", str(0)) + + items, err = listFn(context.Background(), r.Fremote, false, "sub dir/ignore dir") + require.NoError(t, err) + require.Len(t, items, 0) + + items, err = listFn(context.Background(), r.Fremote, true, "sub dir/ignore dir") + require.NoError(t, err) + require.Len(t, items, 2) + assert.Equal(t, "sub dir/ignore dir/.ignore", str(0)) + assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1)) + + fi.Opt.ExcludeFile = nil + items, err = listFn(context.Background(), r.Fremote, false, "sub dir/ignore dir") + require.NoError(t, err) + require.Len(t, items, 2) + assert.Equal(t, "sub dir/ignore dir/.ignore", str(0)) + assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1)) +} + +// TestListDirSorted is integration testing code in fs/list/list.go +// which can't be tested there due to import loops. +func TestListDirSorted(t *testing.T) { + testListDirSorted(t, list.DirSorted) +} + +// TestListDirSortedFn is integration testing code in fs/list/list.go +// which can't be tested there due to import loops. +func TestListDirSortedFn(t *testing.T) { + listFn := func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) { + callback := func(newEntries fs.DirEntries) error { + entries = append(entries, newEntries...) + return nil + } + err = list.DirSortedFn(ctx, f, includeAll, dir, callback, nil) + return entries, err + } + testListDirSorted(t, listFn) +} diff --git a/fs/operations/logger.go b/fs/operations/logger.go new file mode 100644 index 0000000..0d48220 --- /dev/null +++ b/fs/operations/logger.go @@ -0,0 +1,384 @@ +package operations + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + mutex "sync" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/hash" + "github.com/spf13/pflag" +) + +// Sigil represents the rune (-+=*!?) used by Logger to categorize files by their match/differ/missing status. +type Sigil rune + +// String converts sigil to more human-readable string +func (sigil Sigil) String() string { + switch sigil { + case '-': + return "MissingOnSrc" + case '+': + return "MissingOnDst" + case '=': + return "Match" + case '*': + return "Differ" + case '!': + return "Error" + // case '.': + // return "Completed" + case '?': + return "Other" + } + return "unknown" +} + +// Writer directs traffic from sigil -> LoggerOpt.Writer +func (sigil Sigil) Writer(opt LoggerOpt) io.Writer { + switch sigil { + case '-': + return opt.MissingOnSrc + case '+': + return opt.MissingOnDst + case '=': + return opt.Match + case '*': + return opt.Differ + case '!': + return opt.Error + } + return nil +} + +// Sigil constants +const ( + MissingOnSrc Sigil = '-' + MissingOnDst Sigil = '+' + Match Sigil = '=' + Differ Sigil = '*' + TransferError Sigil = '!' + Other Sigil = '?' // reserved but not currently used +) + +// LoggerFn uses fs.DirEntry instead of fs.Object so it can include Dirs +// For LoggerFn example, see bisync.WriteResults() or sync.SyncLoggerFn() +// Usage example: s.logger(ctx, operations.Differ, src, dst, nil) +type LoggerFn func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) +type loggerContextKey struct{} +type loggerOptContextKey struct{} + +var loggerKey = loggerContextKey{} +var loggerOptKey = loggerOptContextKey{} + +// LoggerOpt contains options for the Sync Logger functions +// TODO: refactor Check in here too? +type LoggerOpt struct { + // Fdst, Fsrc fs.Fs // fses to check + // Check checkFn // function to use for checking + // OneWay bool // one way only? + LoggerFn LoggerFn // function to use for logging + Combined io.Writer // a file with file names with leading sigils + MissingOnSrc io.Writer // files only in the destination + MissingOnDst io.Writer // files only in the source + Match io.Writer // matching files + Differ io.Writer // differing files + Error io.Writer // files with errors of some kind + DestAfter io.Writer // files that exist on the destination post-sync + JSON *bytes.Buffer // used by bisync to read/write struct as JSON + DeleteModeOff bool //affects whether Logger expects MissingOnSrc to be deleted + + // lsf options for destAfter + ListFormat ListFormat + JSONOpt ListJSONOpt + LJ *listJSON + Format string + TimeFormat string + Separator string + DirSlash bool + // Recurse bool + HashType hash.Type + FilesOnly bool + DirsOnly bool + Csv bool + Absolute bool +} + +// NewDefaultLoggerFn creates a logger function that writes the sigil and path to configured files that match the sigil +func NewDefaultLoggerFn(opt *LoggerOpt) LoggerFn { + var lock mutex.Mutex + + return func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) { + lock.Lock() + defer lock.Unlock() + + if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil { + opt.PrintDestAfter(ctx, sigil, src, dst, err) + return + } + + _, srcOk := src.(fs.Object) + _, dstOk := dst.(fs.Object) + var filename string + if !srcOk && !dstOk { + return + } else if srcOk && !dstOk { + filename = src.String() + } else { + filename = dst.String() + } + + if sigil.Writer(*opt) != nil { + SyncFprintf(sigil.Writer(*opt), "%s\n", filename) + } + if opt.Combined != nil { + SyncFprintf(opt.Combined, "%c %s\n", sigil, filename) + fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename) + } + if opt.DestAfter != nil { + opt.PrintDestAfter(ctx, sigil, src, dst, err) + } + } +} + +// WithLogger stores logger in ctx and returns a copy of ctx in which loggerKey = logger +func WithLogger(ctx context.Context, logger LoggerFn) context.Context { + return context.WithValue(ctx, loggerKey, logger) +} + +// WithLoggerOpt stores loggerOpt in ctx and returns a copy of ctx in which loggerOptKey = loggerOpt +func WithLoggerOpt(ctx context.Context, loggerOpt LoggerOpt) context.Context { + return context.WithValue(ctx, loggerOptKey, loggerOpt) +} + +// GetLogger attempts to retrieve LoggerFn from context, returns it if found, otherwise returns no-op function +func GetLogger(ctx context.Context) (LoggerFn, bool) { + logger, ok := ctx.Value(loggerKey).(LoggerFn) + if !ok { + logger = func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {} + } + return logger, ok +} + +// GetLoggerOpt attempts to retrieve LoggerOpt from context, returns it if found, otherwise returns NewLoggerOpt() +func GetLoggerOpt(ctx context.Context) LoggerOpt { + loggerOpt, ok := ctx.Value(loggerOptKey).(LoggerOpt) + if ok { + return loggerOpt + } + return NewLoggerOpt() +} + +// WithSyncLogger starts a new logger with the options passed in and saves it to ctx for retrieval later +func WithSyncLogger(ctx context.Context, opt LoggerOpt) context.Context { + ctx = WithLoggerOpt(ctx, opt) + return WithLogger(ctx, func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) { + if opt.LoggerFn != nil { + opt.LoggerFn(ctx, sigil, src, dst, err) + } else { + SyncFprintf(opt.Combined, "%c %s\n", sigil, dst.Remote()) + } + }) +} + +// NewLoggerOpt returns a new LoggerOpt struct with defaults +func NewLoggerOpt() LoggerOpt { + opt := LoggerOpt{ + Combined: new(bytes.Buffer), + MissingOnSrc: new(bytes.Buffer), + MissingOnDst: new(bytes.Buffer), + Match: new(bytes.Buffer), + Differ: new(bytes.Buffer), + Error: new(bytes.Buffer), + DestAfter: new(bytes.Buffer), + JSON: new(bytes.Buffer), + } + return opt +} + +// Winner predicts which side (src or dst) should end up winning out on the dst. +type Winner struct { + Obj fs.DirEntry // the object that should exist on dst post-sync, if any + Side string // whether the winning object was from the src or dst + Err error // whether there's an error preventing us from predicting winner correctly (not whether there was a sync error more generally) +} + +// WinningSide can be called in a LoggerFn to predict what the dest will look like post-sync +// +// This attempts to account for every case in which dst (intentionally) does not match src after a sync. +// +// Known issues / cases we can't confidently predict yet: +// +// --max-duration / CutoffModeHard +// --compare-dest / --copy-dest (because equal() is called multiple times for the same file) +// server-side moves of an entire dir at once (because we never get the individual file objects in the dir) +// High-level retries, because there would be dupes (use --retries 1 to disable) +// Possibly some error scenarios +func WinningSide(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) Winner { + winner := Winner{nil, "none", nil} + opt := GetLoggerOpt(ctx) + ci := fs.GetConfig(ctx) + + if err == fs.ErrorIsDir { + winner.Err = err + if sigil == MissingOnSrc { + if (opt.DeleteModeOff || ci.DryRun) && dst != nil { + winner.Obj = dst + winner.Side = "dst" // whatever's on dst will remain so after DryRun + return winner + } + return winner // none, because dst should just get deleted + } + if sigil == MissingOnDst && ci.DryRun { + return winner // none, because it does not currently exist on dst, and will still not exist after DryRun + } else if ci.DryRun && dst != nil { + winner.Obj = dst + winner.Side = "dst" + } else if src != nil { + winner.Obj = src + winner.Side = "src" + } + return winner + } + + _, srcOk := src.(fs.Object) + _, dstOk := dst.(fs.Object) + if !srcOk && !dstOk { + return winner // none, because we don't have enough info to continue. + } + + switch sigil { + case MissingOnSrc: + if opt.DeleteModeOff || ci.DryRun { // i.e. it's a copy, not sync (or it's a DryRun) + winner.Obj = dst + winner.Side = "dst" // whatever's on dst will remain so after DryRun + return winner + } + return winner // none, because dst should just get deleted + case Match, Differ, MissingOnDst: + if sigil == MissingOnDst && ci.DryRun { + return winner // none, because it does not currently exist on dst, and will still not exist after DryRun + } + winner.Obj = src + winner.Side = "src" // presume dst will end up matching src unless changed below + if sigil == Match && (ci.SizeOnly || ci.CheckSum || ci.IgnoreSize || ci.UpdateOlder || ci.NoUpdateModTime) { + winner.Obj = dst + winner.Side = "dst" // ignore any differences with src because of user flags + } + if ci.IgnoreTimes { + winner.Obj = src + winner.Side = "src" // copy src to dst unconditionally + } + if (sigil == Match || sigil == Differ) && (ci.IgnoreExisting || ci.Immutable) { + winner.Obj = dst + winner.Side = "dst" // dst should remain unchanged if it already exists (and we know it does because it's Match or Differ) + } + if ci.DryRun { + winner.Obj = dst + winner.Side = "dst" // dst should remain unchanged after DryRun (note that we handled MissingOnDst earlier) + } + return winner + case TransferError: + winner.Obj = dst + winner.Side = "dst" // usually, dst should not change if there's an error + if dst == nil { + winner.Obj = src + winner.Side = "src" // but if for some reason we have a src and not a dst, go with it + } + if winner.Obj != nil { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, errors.New("max transfer duration reached as set by --max-duration")) { + winner.Err = err // we can't confidently predict what survives if CutoffModeHard + } + return winner // we know at least one of the objects + } + } + // should only make it this far if it's TransferError and both src and dst are nil + winner.Side = "none" + winner.Err = fmt.Errorf("unknown case -- can't determine winner. %v", err) + fs.Debugf(winner.Obj, "%v", winner.Err) + return winner +} + +// SetListFormat sets opt.ListFormat for destAfter +// TODO: possibly refactor duplicate code from cmd/lsf, where this is mostly copied from +func (opt *LoggerOpt) SetListFormat(ctx context.Context, cmdFlags *pflag.FlagSet) { + // Work out if the separatorFlag was supplied or not + separatorFlag := cmdFlags.Lookup("separator") + separatorFlagSupplied := separatorFlag != nil && separatorFlag.Changed + // Default the separator to , if using CSV + if opt.Csv && !separatorFlagSupplied { + opt.Separator = "," + } + + var list ListFormat + list.SetSeparator(opt.Separator) + list.SetCSV(opt.Csv) + list.SetDirSlash(opt.DirSlash) + list.SetAbsolute(opt.Absolute) + var JSONOpt = ListJSONOpt{ + NoModTime: true, + NoMimeType: true, + DirsOnly: opt.DirsOnly, + FilesOnly: opt.FilesOnly, + // Recurse: opt.Recurse, + } + + for _, char := range opt.Format { + switch char { + case 'p': + list.AddPath() + case 't': + list.AddModTime(opt.TimeFormat) + JSONOpt.NoModTime = false + case 's': + list.AddSize() + case 'h': + list.AddHash(opt.HashType) + JSONOpt.ShowHash = true + JSONOpt.HashTypes = []string{opt.HashType.String()} + case 'i': + list.AddID() + case 'm': + list.AddMimeType() + JSONOpt.NoMimeType = false + case 'e': + list.AddEncrypted() + JSONOpt.ShowEncrypted = true + case 'o': + list.AddOrigID() + JSONOpt.ShowOrigIDs = true + case 'T': + list.AddTier() + case 'M': + list.AddMetadata() + JSONOpt.Metadata = true + default: + fs.Errorf(nil, "unknown format character %q", char) + } + } + opt.ListFormat = list + opt.JSONOpt = JSONOpt +} + +// NewListJSON makes a new *listJSON for destAfter +func (opt *LoggerOpt) NewListJSON(ctx context.Context, fdst fs.Fs, remote string) { + opt.LJ, _ = newListJSON(ctx, fdst, remote, &opt.JSONOpt) + //fs.Debugf(nil, "%v", opt.LJ) +} + +// JSONEntry returns a *ListJSONItem for destAfter +func (opt *LoggerOpt) JSONEntry(ctx context.Context, entry fs.DirEntry) (*ListJSONItem, error) { + return opt.LJ.entry(ctx, entry) +} + +// PrintDestAfter writes a *ListJSONItem to opt.DestAfter +func (opt *LoggerOpt) PrintDestAfter(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) { + entry := WinningSide(ctx, sigil, src, dst, err) + if entry.Obj != nil { + JSONEntry, _ := opt.JSONEntry(ctx, entry.Obj) + _, _ = fmt.Fprintln(opt.DestAfter, opt.ListFormat.Format(JSONEntry)) + } +} diff --git a/fs/operations/lsjson.go b/fs/operations/lsjson.go new file mode 100644 index 0000000..8e714ef --- /dev/null +++ b/fs/operations/lsjson.go @@ -0,0 +1,348 @@ +package operations + +import ( + "context" + "errors" + "fmt" + "path" + "strings" + "time" + + "github.com/rclone/rclone/backend/crypt" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/walk" +) + +// ListJSONItem in the struct which gets marshalled for each line +type ListJSONItem struct { + Path string + Name string + EncryptedPath string `json:",omitempty"` + Encrypted string `json:",omitempty"` + Size int64 + MimeType string `json:",omitempty"` + ModTime Timestamp //`json:",omitempty"` + IsDir bool + Hashes map[string]string `json:",omitempty"` + ID string `json:",omitempty"` + OrigID string `json:",omitempty"` + Tier string `json:",omitempty"` + IsBucket bool `json:",omitempty"` + Metadata fs.Metadata `json:",omitempty"` +} + +// Timestamp a time in the provided format +type Timestamp struct { + When time.Time + Format string +} + +// MarshalJSON turns a Timestamp into JSON +func (t Timestamp) MarshalJSON() (out []byte, err error) { + if t.When.IsZero() { + return []byte(`""`), nil + } + return []byte(`"` + t.When.Format(t.Format) + `"`), nil +} + +// Returns a time format for the given precision +func formatForPrecision(precision time.Duration) string { + switch { + case precision <= time.Nanosecond: + return "2006-01-02T15:04:05.000000000Z07:00" + case precision <= 10*time.Nanosecond: + return "2006-01-02T15:04:05.00000000Z07:00" + case precision <= 100*time.Nanosecond: + return "2006-01-02T15:04:05.0000000Z07:00" + case precision <= time.Microsecond: + return "2006-01-02T15:04:05.000000Z07:00" + case precision <= 10*time.Microsecond: + return "2006-01-02T15:04:05.00000Z07:00" + case precision <= 100*time.Microsecond: + return "2006-01-02T15:04:05.0000Z07:00" + case precision <= time.Millisecond: + return "2006-01-02T15:04:05.000Z07:00" + case precision <= 10*time.Millisecond: + return "2006-01-02T15:04:05.00Z07:00" + case precision <= 100*time.Millisecond: + return "2006-01-02T15:04:05.0Z07:00" + } + return time.RFC3339 +} + +// ListJSONOpt describes the options for ListJSON +type ListJSONOpt struct { + Recurse bool `json:"recurse"` + NoModTime bool `json:"noModTime"` + NoMimeType bool `json:"noMimeType"` + ShowEncrypted bool `json:"showEncrypted"` + ShowOrigIDs bool `json:"showOrigIDs"` + ShowHash bool `json:"showHash"` + DirsOnly bool `json:"dirsOnly"` + FilesOnly bool `json:"filesOnly"` + Metadata bool `json:"metadata"` + HashTypes []string `json:"hashTypes"` // hash types to show if ShowHash is set, e.g. "MD5", "SHA-1" +} + +// state for ListJson +type listJSON struct { + fsrc fs.Fs + remote string + format string + opt *ListJSONOpt + cipher *crypt.Cipher + hashTypes []hash.Type + dirs bool + files bool + canGetTier bool + isBucket bool + showHash bool +} + +func newListJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt) (*listJSON, error) { + lj := &listJSON{ + fsrc: fsrc, + remote: remote, + opt: opt, + dirs: true, + files: true, + } + // Dirs Files + // !FilesOnly,!DirsOnly true true + // !FilesOnly,DirsOnly true false + // FilesOnly,!DirsOnly false true + // FilesOnly,DirsOnly true true + if !opt.FilesOnly && opt.DirsOnly { + lj.files = false + } else if opt.FilesOnly && !opt.DirsOnly { + lj.dirs = false + } + if opt.ShowEncrypted { + fsInfo, _, _, config, err := fs.ConfigFs(fs.ConfigStringFull(fsrc)) + if err != nil { + return nil, fmt.Errorf("ListJSON failed to load config for crypt remote: %w", err) + } + if fsInfo.Name != "crypt" { + return nil, errors.New("the remote needs to be of type \"crypt\"") + } + lj.cipher, err = crypt.NewCipher(config) + if err != nil { + return nil, fmt.Errorf("ListJSON failed to make new crypt remote: %w", err) + } + } + features := fsrc.Features() + lj.canGetTier = features.GetTier + lj.format = formatForPrecision(fsrc.Precision()) + lj.isBucket = features.BucketBased && remote == "" && fsrc.Root() == "" // if bucket-based remote listing the root mark directories as buckets + lj.showHash = opt.ShowHash + lj.hashTypes = fsrc.Hashes().Array() + if len(opt.HashTypes) != 0 { + lj.showHash = true + lj.hashTypes = []hash.Type{} + for _, hashType := range opt.HashTypes { + var ht hash.Type + err := ht.Set(hashType) + if err != nil { + return nil, err + } + lj.hashTypes = append(lj.hashTypes, ht) + } + } + return lj, nil +} + +// Convert a single entry to JSON +// +// It may return nil if there is no entry to return +func (lj *listJSON) entry(ctx context.Context, entry fs.DirEntry) (*ListJSONItem, error) { + switch entry.(type) { + case fs.Directory: + if lj.opt.FilesOnly { + return nil, nil + } + case fs.Object: + if lj.opt.DirsOnly { + return nil, nil + } + default: + fs.Errorf(nil, "Unknown type %T in listing", entry) + } + + item := &ListJSONItem{ + Path: entry.Remote(), + Name: path.Base(entry.Remote()), + Size: entry.Size(), + } + if entry.Remote() == "" { + item.Name = "" + } + if !lj.opt.NoModTime { + item.ModTime = Timestamp{When: entry.ModTime(ctx), Format: lj.format} + } + if !lj.opt.NoMimeType { + item.MimeType = fs.MimeTypeDirEntry(ctx, entry) + } + if lj.cipher != nil { + switch entry.(type) { + case fs.Directory: + item.EncryptedPath = lj.cipher.EncryptDirName(entry.Remote()) + case fs.Object: + item.EncryptedPath = lj.cipher.EncryptFileName(entry.Remote()) + default: + fs.Errorf(nil, "Unknown type %T in listing", entry) + } + item.Encrypted = path.Base(item.EncryptedPath) + } + if lj.opt.Metadata { + metadata, err := fs.GetMetadata(ctx, entry) + if err != nil { + fs.Errorf(entry, "Failed to read metadata: %v", err) + } else if metadata != nil { + item.Metadata = metadata + } + } + if do, ok := entry.(fs.IDer); ok { + item.ID = do.ID() + } + if o, ok := entry.(fs.Object); lj.opt.ShowOrigIDs && ok { + if do, ok := fs.UnWrapObject(o).(fs.IDer); ok { + item.OrigID = do.ID() + } + } + switch x := entry.(type) { + case fs.Directory: + item.IsDir = true + item.IsBucket = lj.isBucket + case fs.Object: + item.IsDir = false + if lj.showHash { + item.Hashes = make(map[string]string) + for _, hashType := range lj.hashTypes { + hash, err := x.Hash(ctx, hashType) + if err != nil { + fs.Errorf(x, "Failed to read hash: %v", err) + } else if hash != "" { + item.Hashes[hashType.String()] = hash + } + } + } + if lj.canGetTier { + if do, ok := x.(fs.GetTierer); ok { + item.Tier = do.GetTier() + } + } + default: + fs.Errorf(nil, "Unknown type %T in listing in ListJSON", entry) + } + return item, nil +} + +// ListJSON lists fsrc using the options in opt calling callback for each item +func ListJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt, callback func(*ListJSONItem) error) error { + lj, err := newListJSON(ctx, fsrc, remote, opt) + if err != nil { + return err + } + err = walk.ListR(ctx, fsrc, remote, false, ConfigMaxDepth(ctx, lj.opt.Recurse), walk.ListAll, func(entries fs.DirEntries) (err error) { + for _, entry := range entries { + item, err := lj.entry(ctx, entry) + if err != nil { + return fmt.Errorf("creating entry failed in ListJSON: %w", err) + } + if item != nil { + err = callback(item) + if err != nil { + return fmt.Errorf("callback failed in ListJSON: %w", err) + } + } + } + return nil + }) + if err != nil { + return fmt.Errorf("error in ListJSON: %w", err) + } + return nil +} + +// StatJSON returns a single JSON stat entry for the fsrc, remote path +// +// The item returned may be nil if it is not found or excluded with DirsOnly/FilesOnly +func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt) (item *ListJSONItem, err error) { + // FIXME this could me more efficient we had a new primitive + // NewDirEntry() which returned an Object or a Directory + lj, err := newListJSON(ctx, fsrc, remote, opt) + if err != nil { + return nil, err + } + + // Root is always a directory. When we have a NewDirEntry + // primitive we need to call it, but for now this will do. + if remote == "" { + if !lj.dirs { + return nil, nil + } + // Check the root directory exists + entries, err := fsrc.List(ctx, "") + accounting.Stats(ctx).Listed(int64(len(entries))) + if err != nil { + return nil, err + } + return lj.entry(ctx, fs.NewDir("", time.Now())) + } + + // Could be a file or a directory here + if lj.files && !strings.HasSuffix(remote, "/") { + // NewObject can return the sentinel errors ErrorObjectNotFound or ErrorIsDir + // ErrorObjectNotFound can mean the source is a directory or not found + obj, err := fsrc.NewObject(ctx, remote) + if err == fs.ErrorObjectNotFound { + if !lj.dirs { + return nil, nil + } + } else if err == fs.ErrorIsDir { + if !lj.dirs { + return nil, nil + } + // This could return a made up ListJSONItem here + // but that wouldn't have the IDs etc in + } else if err != nil { + if !lj.dirs { + return nil, err + } + } else { + return lj.entry(ctx, obj) + } + } + // Must be a directory here + // + // Remove trailing / as rclone listings won't have them + remote = strings.TrimRight(remote, "/") + parent := path.Dir(remote) + if parent == "." || parent == "/" { + parent = "" + } + entries, err := fsrc.List(ctx, parent) + accounting.Stats(ctx).Listed(int64(len(entries))) + if err == fs.ErrorDirNotFound { + return nil, nil + } else if err != nil { + return nil, err + } + equal := func(a, b string) bool { return a == b } + if fsrc.Features().CaseInsensitive { + equal = strings.EqualFold + } + var foundEntry fs.DirEntry + for _, entry := range entries { + if equal(entry.Remote(), remote) { + foundEntry = entry + break + } + } + if foundEntry == nil { + return nil, nil + } + return lj.entry(ctx, foundEntry) +} diff --git a/fs/operations/lsjson_test.go b/fs/operations/lsjson_test.go new file mode 100644 index 0000000..75d4728 --- /dev/null +++ b/fs/operations/lsjson_test.go @@ -0,0 +1,406 @@ +package operations_test + +import ( + "context" + "sort" + "testing" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fstest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Compare a and b in a file system independent way +func compareListJSONItem(t *testing.T, a, b *operations.ListJSONItem, precision time.Duration) { + assert.Equal(t, a.Path, b.Path, "Path") + assert.Equal(t, a.Name, b.Name, "Name") + // assert.Equal(t, a.EncryptedPath, b.EncryptedPath, "EncryptedPath") + // assert.Equal(t, a.Encrypted, b.Encrypted, "Encrypted") + if !a.IsDir { + assert.Equal(t, a.Size, b.Size, "Size") + } + // assert.Equal(t, a.MimeType, a.Mib.MimeType, "MimeType") + if !a.IsDir { + fstest.AssertTimeEqualWithPrecision(t, "ListJSON", a.ModTime.When, b.ModTime.When, precision) + } + assert.Equal(t, a.IsDir, b.IsDir, "IsDir") + // assert.Equal(t, a.Hashes, a.b.Hashes, "Hashes") + // assert.Equal(t, a.ID, b.ID, "ID") + // assert.Equal(t, a.OrigID, a.b.OrigID, "OrigID") + // assert.Equal(t, a.Tier, b.Tier, "Tier") + // assert.Equal(t, a.IsBucket, a.Isb.IsBucket, "IsBucket") +} + +func TestListJSON(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteBoth(ctx, "file1", "file1", t1) + file2 := r.WriteBoth(ctx, "sub/file2", "sub/file2", t2) + + r.CheckRemoteItems(t, file1, file2) + precision := fs.GetModifyWindow(ctx, r.Fremote) + + for _, test := range []struct { + name string + remote string + opt operations.ListJSONOpt + want []*operations.ListJSONItem + }{ + { + name: "Default", + opt: operations.ListJSONOpt{}, + want: []*operations.ListJSONItem{{ + Path: "file1", + Name: "file1", + Size: 5, + ModTime: operations.Timestamp{When: t1}, + IsDir: false, + }, { + Path: "sub", + Name: "sub", + IsDir: true, + }}, + }, { + name: "FilesOnly", + opt: operations.ListJSONOpt{ + FilesOnly: true, + }, + want: []*operations.ListJSONItem{{ + Path: "file1", + Name: "file1", + Size: 5, + ModTime: operations.Timestamp{When: t1}, + IsDir: false, + }}, + }, { + name: "DirsOnly", + opt: operations.ListJSONOpt{ + DirsOnly: true, + }, + want: []*operations.ListJSONItem{{ + Path: "sub", + Name: "sub", + IsDir: true, + }}, + }, { + name: "Recurse", + opt: operations.ListJSONOpt{ + Recurse: true, + }, + want: []*operations.ListJSONItem{{ + Path: "file1", + Name: "file1", + Size: 5, + ModTime: operations.Timestamp{When: t1}, + IsDir: false, + }, { + Path: "sub", + Name: "sub", + IsDir: true, + }, { + Path: "sub/file2", + Name: "file2", + Size: 9, + ModTime: operations.Timestamp{When: t2}, + IsDir: false, + }}, + }, { + name: "SubDir", + remote: "sub", + opt: operations.ListJSONOpt{}, + want: []*operations.ListJSONItem{{ + Path: "sub/file2", + Name: "file2", + Size: 9, + ModTime: operations.Timestamp{When: t2}, + IsDir: false, + }}, + }, { + name: "NoModTime", + opt: operations.ListJSONOpt{ + FilesOnly: true, + NoModTime: true, + }, + want: []*operations.ListJSONItem{{ + Path: "file1", + Name: "file1", + Size: 5, + ModTime: operations.Timestamp{When: time.Time{}}, + IsDir: false, + }}, + }, { + name: "NoMimeType", + opt: operations.ListJSONOpt{ + FilesOnly: true, + NoMimeType: true, + }, + want: []*operations.ListJSONItem{{ + Path: "file1", + Name: "file1", + Size: 5, + ModTime: operations.Timestamp{When: t1}, + IsDir: false, + }}, + }, { + name: "ShowHash", + opt: operations.ListJSONOpt{ + FilesOnly: true, + ShowHash: true, + }, + want: []*operations.ListJSONItem{{ + Path: "file1", + Name: "file1", + Size: 5, + ModTime: operations.Timestamp{When: t1}, + IsDir: false, + }}, + }, { + name: "HashTypes", + opt: operations.ListJSONOpt{ + FilesOnly: true, + ShowHash: true, + HashTypes: []string{"MD5"}, + }, + want: []*operations.ListJSONItem{{ + Path: "file1", + Name: "file1", + Size: 5, + ModTime: operations.Timestamp{When: t1}, + IsDir: false, + }}, + }, { + name: "Metadata", + opt: operations.ListJSONOpt{ + FilesOnly: false, + Metadata: true, + }, + want: []*operations.ListJSONItem{{ + Path: "file1", + Name: "file1", + Size: 5, + ModTime: operations.Timestamp{When: t1}, + IsDir: false, + }, { + Path: "sub", + Name: "sub", + IsDir: true, + }}, + }, + } { + t.Run(test.name, func(t *testing.T) { + var got []*operations.ListJSONItem + require.NoError(t, operations.ListJSON(ctx, r.Fremote, test.remote, &test.opt, func(item *operations.ListJSONItem) error { + got = append(got, item) + return nil + })) + sort.Slice(got, func(i, j int) bool { + return got[i].Path < got[j].Path + }) + require.Equal(t, len(test.want), len(got), "Wrong number of results") + for i := range test.want { + compareListJSONItem(t, test.want[i], got[i], precision) + if test.opt.NoMimeType { + assert.Equal(t, "", got[i].MimeType) + } else { + assert.NotEqual(t, "", got[i].MimeType) + } + if test.opt.Metadata { + features := r.Fremote.Features() + if features.ReadMetadata && !got[i].IsDir { + assert.Greater(t, len(got[i].Metadata), 0, "Expecting metadata for file") + } + if features.ReadDirMetadata && got[i].IsDir { + assert.Greater(t, len(got[i].Metadata), 0, "Expecting metadata for dir") + } + } + if test.opt.ShowHash { + hashes := got[i].Hashes + assert.NotNil(t, hashes) + if len(test.opt.HashTypes) > 0 && len(hashes) > 0 { + assert.Equal(t, 1, len(hashes)) + } + if hashes["crc32"] != "" { + assert.Equal(t, "9ee760e5", hashes["crc32"]) + } + if hashes["dropbox"] != "" { + assert.Equal(t, "f4d62afeaee6f35d3efdd8c66623360395165473bcc958f835343eb3f542f983", hashes["dropbox"]) + } + if hashes["mailru"] != "" { + assert.Equal(t, "66696c6531000000000000000000000000000000", hashes["mailru"]) + } + if hashes["md5"] != "" { + assert.Equal(t, "826e8142e6baabe8af779f5f490cf5f5", hashes["md5"]) + } + if hashes["quickxor"] != "" { + assert.Equal(t, "6648031bca100300000000000500000000000000", hashes["quickxor"]) + } + if hashes["sha1"] != "" { + assert.Equal(t, "60b27f004e454aca81b0480209cce5081ec52390", hashes["sha1"]) + } + if hashes["sha256"] != "" { + assert.Equal(t, "c147efcfc2d7ea666a9e4f5187b115c90903f0fc896a56df9a6ef5d8f3fc9f31", hashes["sha256"]) + } + if hashes["whirlpool"] != "" { + assert.Equal(t, "02fa11755b6470bfc5aab6d94cde5cf2939474fb5b0ebbf8ddf3d32bf06aa438eb92eac097047c02017dc1c317ee83fa8a2717ca4d544b4ee75b3231d1c466b0", hashes["whirlpool"]) + } + } else { + assert.Nil(t, got[i].Hashes) + } + } + }) + } +} + +func TestStatJSON(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteBoth(ctx, "file1", "file1", t1) + file2 := r.WriteBoth(ctx, "sub/file2", "sub/file2", t2) + + r.CheckRemoteItems(t, file1, file2) + precision := fs.GetModifyWindow(ctx, r.Fremote) + + for _, test := range []struct { + name string + remote string + opt operations.ListJSONOpt + want *operations.ListJSONItem + }{ + { + name: "Root", + remote: "", + opt: operations.ListJSONOpt{}, + want: &operations.ListJSONItem{ + Path: "", + Name: "", + IsDir: true, + }, + }, { + name: "RootFilesOnly", + remote: "", + opt: operations.ListJSONOpt{ + FilesOnly: true, + }, + want: nil, + }, { + name: "RootDirsOnly", + remote: "", + opt: operations.ListJSONOpt{ + DirsOnly: true, + }, + want: &operations.ListJSONItem{ + Path: "", + Name: "", + IsDir: true, + }, + }, { + name: "Dir", + remote: "sub", + opt: operations.ListJSONOpt{}, + want: &operations.ListJSONItem{ + Path: "sub", + Name: "sub", + IsDir: true, + }, + }, { + name: "DirWithTrailingSlash", + remote: "sub/", + opt: operations.ListJSONOpt{}, + want: &operations.ListJSONItem{ + Path: "sub", + Name: "sub", + IsDir: true, + }, + }, { + name: "File", + remote: "file1", + opt: operations.ListJSONOpt{}, + want: &operations.ListJSONItem{ + Path: "file1", + Name: "file1", + Size: 5, + ModTime: operations.Timestamp{When: t1}, + IsDir: false, + }, + }, { + name: "NotFound", + remote: "notfound", + opt: operations.ListJSONOpt{}, + want: nil, + }, { + name: "DirFilesOnly", + remote: "sub", + opt: operations.ListJSONOpt{ + FilesOnly: true, + }, + want: nil, + }, { + name: "FileFilesOnly", + remote: "file1", + opt: operations.ListJSONOpt{ + FilesOnly: true, + }, + want: &operations.ListJSONItem{ + Path: "file1", + Name: "file1", + Size: 5, + ModTime: operations.Timestamp{When: t1}, + IsDir: false, + }, + }, { + name: "NotFoundFilesOnly", + remote: "notfound", + opt: operations.ListJSONOpt{ + FilesOnly: true, + }, + want: nil, + }, { + name: "DirDirsOnly", + remote: "sub", + opt: operations.ListJSONOpt{ + DirsOnly: true, + }, + want: &operations.ListJSONItem{ + Path: "sub", + Name: "sub", + IsDir: true, + }, + }, { + name: "FileDirsOnly", + remote: "file1", + opt: operations.ListJSONOpt{ + DirsOnly: true, + }, + want: nil, + }, { + name: "NotFoundDirsOnly", + remote: "notfound", + opt: operations.ListJSONOpt{ + DirsOnly: true, + }, + want: nil, + }, + } { + t.Run(test.name, func(t *testing.T) { + got, err := operations.StatJSON(ctx, r.Fremote, test.remote, &test.opt) + require.NoError(t, err) + if test.want == nil { + assert.Nil(t, got) + return + } + require.NotNil(t, got) + compareListJSONItem(t, test.want, got, precision) + }) + } + + t.Run("RootNotFound", func(t *testing.T) { + f, err := fs.NewFs(ctx, r.FremoteName+"/notfound") + require.NoError(t, err) + _, err = operations.StatJSON(ctx, f, "", &operations.ListJSONOpt{}) + // This should return an error except for bucket based remotes + assert.True(t, err != nil || f.Features().BucketBased, "Need an error for non bucket based backends") + }) +} diff --git a/fs/operations/multithread.go b/fs/operations/multithread.go new file mode 100644 index 0000000..9abd5f7 --- /dev/null +++ b/fs/operations/multithread.go @@ -0,0 +1,380 @@ +package operations + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/lib/atexit" + "github.com/rclone/rclone/lib/multipart" + "github.com/rclone/rclone/lib/pool" + "golang.org/x/sync/errgroup" +) + +const ( + multithreadChunkSize = 64 << 10 +) + +// Return a boolean as to whether we should use multi thread copy for +// this transfer +func doMultiThreadCopy(ctx context.Context, f fs.Fs, src fs.Object) bool { + ci := fs.GetConfig(ctx) + + // Disable multi thread if... + + // ...it isn't configured + if ci.MultiThreadStreams <= 1 { + return false + } + // ...if the source doesn't support it + if src.Fs().Features().NoMultiThreading { + return false + } + // ...size of object is less than cutoff + if src.Size() < int64(ci.MultiThreadCutoff) { + return false + } + // ...destination doesn't support it + dstFeatures := f.Features() + if dstFeatures.OpenChunkWriter == nil && dstFeatures.OpenWriterAt == nil { + return false + } + // ...if --multi-thread-streams not in use and source and + // destination are both local + if !ci.MultiThreadSet && dstFeatures.IsLocal && src.Fs().Features().IsLocal { + return false + } + return true +} + +// state for a multi-thread copy +type multiThreadCopyState struct { + ctx context.Context + partSize int64 + size int64 + src fs.Object + acc *accounting.Account + numChunks int + noBuffering bool // set to read the input without buffering +} + +// Copy a single chunk into place +func (mc *multiThreadCopyState) copyChunk(ctx context.Context, chunk int, writer fs.ChunkWriter) (err error) { + defer func() { + if err != nil { + fs.Debugf(mc.src, "multi-thread copy: chunk %d/%d failed: %v", chunk+1, mc.numChunks, err) + } + }() + start := int64(chunk) * mc.partSize + if start >= mc.size { + return nil + } + end := min(start+mc.partSize, mc.size) + size := end - start + + // Reserve the memory first so we don't open the source and wait for memory buffers for ages + var rw *pool.RW + if !mc.noBuffering { + rw = multipart.NewRW().Reserve(size) + defer fs.CheckClose(rw, &err) + } + + fs.Debugf(mc.src, "multi-thread copy: chunk %d/%d (%d-%d) size %v starting", chunk+1, mc.numChunks, start, end, fs.SizeSuffix(size)) + + rc, err := Open(ctx, mc.src, &fs.RangeOption{Start: start, End: end - 1}) + if err != nil { + return fmt.Errorf("multi-thread copy: failed to open source: %w", err) + } + defer fs.CheckClose(rc, &err) + + var rs io.ReadSeeker + if mc.noBuffering { + // Read directly if we are sure we aren't going to seek + // and account with accounting + rc.SetAccounting(mc.acc.AccountRead) + rs = rc + } else { + // Read the chunk into buffered reader + _, err = io.CopyN(rw, rc, size) + if err != nil { + return fmt.Errorf("multi-thread copy: failed to read chunk: %w", err) + } + // Account as we go + rw.SetAccounting(mc.acc.AccountRead) + rs = rw + } + + // Write the chunk + bytesWritten, err := writer.WriteChunk(ctx, chunk, rs) + if err != nil { + return fmt.Errorf("multi-thread copy: failed to write chunk: %w", err) + } + + fs.Debugf(mc.src, "multi-thread copy: chunk %d/%d (%d-%d) size %v finished", chunk+1, mc.numChunks, start, end, fs.SizeSuffix(bytesWritten)) + return nil +} + +// Given a file size and a chunkSize +// it returns the number of chunks, so that chunkSize * numChunks >= size +func calculateNumChunks(size int64, chunkSize int64) int { + numChunks := size / chunkSize + if size%chunkSize != 0 { + numChunks++ + } + return int(numChunks) +} + +// Copy src to (f, remote) using streams download threads. It tries to use the OpenChunkWriter feature +// and if that's not available it creates an adapter using OpenWriterAt +func multiThreadCopy(ctx context.Context, f fs.Fs, remote string, src fs.Object, concurrency int, tr *accounting.Transfer, options ...fs.OpenOption) (newDst fs.Object, err error) { + openChunkWriter := f.Features().OpenChunkWriter + ci := fs.GetConfig(ctx) + noBuffering := false + usingOpenWriterAt := false + if openChunkWriter == nil { + openWriterAt := f.Features().OpenWriterAt + if openWriterAt == nil { + return nil, errors.New("multi-thread copy: neither OpenChunkWriter nor OpenWriterAt supported") + } + openChunkWriter = openChunkWriterFromOpenWriterAt(openWriterAt, int64(ci.MultiThreadChunkSize), int64(ci.MultiThreadWriteBufferSize), f) + // If we are using OpenWriterAt we don't seek the chunks so don't need to buffer + fs.Debugf(src, "multi-thread copy: disabling buffering because destination uses OpenWriterAt") + noBuffering = true + usingOpenWriterAt = true + } else if src.Fs().Features().IsLocal { + // If the source fs is local we don't need to buffer + fs.Debugf(src, "multi-thread copy: disabling buffering because source is local disk") + noBuffering = true + } else if f.Features().ChunkWriterDoesntSeek { + // If the destination Fs promises not to seek its chunks + // (except for retries) then we don't need buffering. + fs.Debugf(src, "multi-thread copy: disabling buffering because destination has set ChunkWriterDoesntSeek") + noBuffering = true + } + + if src.Size() < 0 { + return nil, fmt.Errorf("multi-thread copy: can't copy unknown sized file") + } + if src.Size() == 0 { + return nil, fmt.Errorf("multi-thread copy: can't copy zero sized file") + } + + info, chunkWriter, err := openChunkWriter(ctx, remote, src, options...) + if err != nil { + return nil, fmt.Errorf("multi-thread copy: failed to open chunk writer: %w", err) + } + + uploadCtx, cancel := context.WithCancel(ctx) + defer cancel() + uploadedOK := false + defer atexit.OnError(&err, func() { + cancel() + if info.LeavePartsOnError || uploadedOK { + return + } + fs.Debugf(src, "multi-thread copy: cancelling transfer on exit") + abortErr := chunkWriter.Abort(ctx) + if abortErr != nil { + fs.Debugf(src, "multi-thread copy: abort failed: %v", abortErr) + } + })() + + if info.ChunkSize > src.Size() { + fs.Debugf(src, "multi-thread copy: chunk size %v was bigger than source file size %v", fs.SizeSuffix(info.ChunkSize), fs.SizeSuffix(src.Size())) + info.ChunkSize = src.Size() + } + + // Use the backend concurrency if it is higher than --multi-thread-streams or if --multi-thread-streams wasn't set explicitly + if !ci.MultiThreadSet || info.Concurrency > concurrency { + fs.Debugf(src, "multi-thread copy: using backend concurrency of %d instead of --multi-thread-streams %d", info.Concurrency, concurrency) + concurrency = info.Concurrency + } + + numChunks := calculateNumChunks(src.Size(), info.ChunkSize) + if concurrency > numChunks { + fs.Debugf(src, "multi-thread copy: number of streams %d was bigger than number of chunks %d", concurrency, numChunks) + concurrency = numChunks + } + + if concurrency < 1 { + concurrency = 1 + } + + g, gCtx := errgroup.WithContext(uploadCtx) + g.SetLimit(concurrency) + + mc := &multiThreadCopyState{ + ctx: gCtx, + size: src.Size(), + src: src, + partSize: info.ChunkSize, + numChunks: numChunks, + noBuffering: noBuffering, + } + + // Make accounting + mc.acc = tr.Account(gCtx, nil) + + fs.Debugf(src, "Starting multi-thread copy with %d chunks of size %v with %v parallel streams", mc.numChunks, fs.SizeSuffix(mc.partSize), concurrency) + for chunk := range mc.numChunks { + // Fail fast, in case an errgroup managed function returns an error + if gCtx.Err() != nil { + break + } + chunk := chunk + g.Go(func() error { + return mc.copyChunk(gCtx, chunk, chunkWriter) + }) + } + + err = g.Wait() + if err != nil { + return nil, err + } + err = chunkWriter.Close(ctx) + if err != nil { + return nil, fmt.Errorf("multi-thread copy: failed to close object after copy: %w", err) + } + uploadedOK = true // file is definitely uploaded OK so no need to abort + + obj, err := f.NewObject(ctx, remote) + if err != nil { + return nil, fmt.Errorf("multi-thread copy: failed to find object after copy: %w", err) + } + + // OpenWriterAt doesn't set metadata so we need to set it on completion + if usingOpenWriterAt { + setModTime := true + if ci.Metadata { + do, ok := obj.(fs.SetMetadataer) + if ok { + meta, err := fs.GetMetadataOptions(ctx, f, src, options) + if err != nil { + return nil, fmt.Errorf("multi-thread copy: failed to read metadata from source object: %w", err) + } + if _, foundMeta := meta["mtime"]; !foundMeta { + meta.Set("mtime", src.ModTime(ctx).Format(time.RFC3339Nano)) + } + err = do.SetMetadata(ctx, meta) + if err != nil { + return nil, fmt.Errorf("multi-thread copy: failed to set metadata: %w", err) + } + setModTime = false + } else { + fs.Errorf(obj, "multi-thread copy: can't set metadata as SetMetadata isn't implemented in: %v", f) + } + } + if setModTime { + err = obj.SetModTime(ctx, src.ModTime(ctx)) + switch err { + case nil, fs.ErrorCantSetModTime, fs.ErrorCantSetModTimeWithoutDelete: + default: + return nil, fmt.Errorf("multi-thread copy: failed to set modification time: %w", err) + } + } + } + + fs.Debugf(src, "Finished multi-thread copy with %d parts of size %v", mc.numChunks, fs.SizeSuffix(mc.partSize)) + return obj, nil +} + +// writerAtChunkWriter converts a WriterAtCloser into a ChunkWriter +type writerAtChunkWriter struct { + remote string + size int64 + writerAt fs.WriterAtCloser + chunkSize int64 + chunks int + writeBufferSize int64 + f fs.Fs + closed bool +} + +// WriteChunk writes chunkNumber from reader +func (w *writerAtChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (int64, error) { + fs.Debugf(w.remote, "writing chunk %v", chunkNumber) + + bytesToWrite := w.chunkSize + if chunkNumber == (w.chunks-1) && w.size%w.chunkSize != 0 { + bytesToWrite = w.size % w.chunkSize + } + + var writer io.Writer = io.NewOffsetWriter(w.writerAt, int64(chunkNumber)*w.chunkSize) + if w.writeBufferSize > 0 { + writer = bufio.NewWriterSize(writer, int(w.writeBufferSize)) + } + n, err := io.Copy(writer, reader) + if err != nil { + return -1, err + } + if n != bytesToWrite { + return -1, fmt.Errorf("expected to write %v bytes for chunk %v, but wrote %v bytes", bytesToWrite, chunkNumber, n) + } + // if we were buffering, flush to disk + switch w := writer.(type) { + case *bufio.Writer: + err = w.Flush() + if err != nil { + return -1, fmt.Errorf("multi-thread copy: flush failed: %w", err) + } + } + return n, nil +} + +// Close the chunk writing +func (w *writerAtChunkWriter) Close(ctx context.Context) error { + if w.closed { + return nil + } + w.closed = true + return w.writerAt.Close() +} + +// Abort the chunk writing +func (w *writerAtChunkWriter) Abort(ctx context.Context) error { + err := w.Close(ctx) + if err != nil { + fs.Errorf(w.remote, "multi-thread copy: failed to close file before aborting: %v", err) + } + obj, err := w.f.NewObject(ctx, w.remote) + if err != nil { + return fmt.Errorf("multi-thread copy: failed to find temp file when aborting chunk writer: %w", err) + } + return obj.Remove(ctx) +} + +// openChunkWriterFromOpenWriterAt adapts an OpenWriterAtFn into an OpenChunkWriterFn using chunkSize and writeBufferSize +func openChunkWriterFromOpenWriterAt(openWriterAt fs.OpenWriterAtFn, chunkSize int64, writeBufferSize int64, f fs.Fs) fs.OpenChunkWriterFn { + return func(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) { + ci := fs.GetConfig(ctx) + + writerAt, err := openWriterAt(ctx, remote, src.Size()) + if err != nil { + return info, nil, err + } + + if writeBufferSize > 0 { + fs.Debugf(src.Remote(), "multi-thread copy: write buffer set to %v", writeBufferSize) + } + + chunkWriter := &writerAtChunkWriter{ + remote: remote, + size: src.Size(), + chunkSize: chunkSize, + chunks: calculateNumChunks(src.Size(), chunkSize), + writerAt: writerAt, + writeBufferSize: writeBufferSize, + f: f, + } + info = fs.ChunkWriterInfo{ + ChunkSize: chunkSize, + Concurrency: ci.MultiThreadStreams, + } + return info, chunkWriter, nil + } +} diff --git a/fs/operations/multithread_test.go b/fs/operations/multithread_test.go new file mode 100644 index 0000000..d3a07ae --- /dev/null +++ b/fs/operations/multithread_test.go @@ -0,0 +1,333 @@ +package operations + +import ( + "context" + "errors" + "fmt" + "io" + "sync" + "testing" + "time" + + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/object" + "github.com/rclone/rclone/fstest/mockfs" + "github.com/rclone/rclone/fstest/mockobject" + "github.com/rclone/rclone/lib/random" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fstest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDoMultiThreadCopy(t *testing.T) { + ctx := context.Background() + ci := fs.GetConfig(ctx) + f, err := mockfs.NewFs(ctx, "potato", "", nil) + require.NoError(t, err) + src := mockobject.New("file.txt").WithContent([]byte(random.String(100)), mockobject.SeekModeNone) + srcFs, err := mockfs.NewFs(ctx, "sausage", "", nil) + require.NoError(t, err) + src.SetFs(srcFs) + + oldStreams := ci.MultiThreadStreams + oldCutoff := ci.MultiThreadCutoff + oldIsSet := ci.MultiThreadSet + defer func() { + ci.MultiThreadStreams = oldStreams + ci.MultiThreadCutoff = oldCutoff + ci.MultiThreadSet = oldIsSet + }() + + ci.MultiThreadStreams, ci.MultiThreadCutoff = 4, 50 + ci.MultiThreadSet = false + + nullWriterAt := func(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) { + panic("don't call me") + } + f.Features().OpenWriterAt = nullWriterAt + + assert.True(t, doMultiThreadCopy(ctx, f, src)) + + ci.MultiThreadStreams = 0 + assert.False(t, doMultiThreadCopy(ctx, f, src)) + ci.MultiThreadStreams = 1 + assert.False(t, doMultiThreadCopy(ctx, f, src)) + ci.MultiThreadStreams = 2 + assert.True(t, doMultiThreadCopy(ctx, f, src)) + + ci.MultiThreadCutoff = 200 + assert.False(t, doMultiThreadCopy(ctx, f, src)) + ci.MultiThreadCutoff = 101 + assert.False(t, doMultiThreadCopy(ctx, f, src)) + ci.MultiThreadCutoff = 100 + assert.True(t, doMultiThreadCopy(ctx, f, src)) + + f.Features().OpenWriterAt = nil + assert.False(t, doMultiThreadCopy(ctx, f, src)) + f.Features().OpenWriterAt = nullWriterAt + assert.True(t, doMultiThreadCopy(ctx, f, src)) + + f.Features().IsLocal = true + srcFs.Features().IsLocal = true + assert.False(t, doMultiThreadCopy(ctx, f, src)) + ci.MultiThreadSet = true + assert.True(t, doMultiThreadCopy(ctx, f, src)) + ci.MultiThreadSet = false + assert.False(t, doMultiThreadCopy(ctx, f, src)) + srcFs.Features().IsLocal = false + assert.True(t, doMultiThreadCopy(ctx, f, src)) + srcFs.Features().IsLocal = true + assert.False(t, doMultiThreadCopy(ctx, f, src)) + f.Features().IsLocal = false + assert.True(t, doMultiThreadCopy(ctx, f, src)) + srcFs.Features().IsLocal = false + assert.True(t, doMultiThreadCopy(ctx, f, src)) + + srcFs.Features().NoMultiThreading = true + assert.False(t, doMultiThreadCopy(ctx, f, src)) + srcFs.Features().NoMultiThreading = false + assert.True(t, doMultiThreadCopy(ctx, f, src)) +} + +func TestMultithreadCalculateNumChunks(t *testing.T) { + for _, test := range []struct { + size int64 + chunkSize int64 + wantNumChunks int + }{ + {size: 1, chunkSize: multithreadChunkSize, wantNumChunks: 1}, + {size: 1 << 20, chunkSize: 1, wantNumChunks: 1 << 20}, + {size: 1 << 20, chunkSize: 2, wantNumChunks: 1 << 19}, + {size: (1 << 20) + 1, chunkSize: 2, wantNumChunks: (1 << 19) + 1}, + {size: (1 << 20) - 1, chunkSize: 2, wantNumChunks: 1 << 19}, + } { + t.Run(fmt.Sprintf("%+v", test), func(t *testing.T) { + mc := &multiThreadCopyState{} + mc.numChunks = calculateNumChunks(test.size, test.chunkSize) + assert.Equal(t, test.wantNumChunks, mc.numChunks) + }) + } +} + +// Skip if not multithread, returning the chunkSize otherwise +func skipIfNotMultithread(ctx context.Context, t *testing.T, r *fstest.Run) int { + features := r.Fremote.Features() + if features.OpenChunkWriter == nil && features.OpenWriterAt == nil { + t.Skip("multithread writing not supported") + } + + // Only support one hash for the local backend otherwise we end up spending a huge amount of CPU on hashing! + if r.Fremote.Features().IsLocal { + oldHashes := hash.SupportOnly([]hash.Type{r.Fremote.Hashes().GetOne()}) + t.Cleanup(func() { + _ = hash.SupportOnly(oldHashes) + }) + } + + ci := fs.GetConfig(ctx) + chunkSize := int(ci.MultiThreadChunkSize) + if features.OpenChunkWriter != nil { + //OpenChunkWriter func(ctx context.Context, remote string, src ObjectInfo, options ...OpenOption) (info ChunkWriterInfo, writer ChunkWriter, err error) + const fileName = "chunksize-probe" + src := object.NewStaticObjectInfo(fileName, time.Now(), int64(100*fs.Mebi), true, nil, nil) + info, writer, err := features.OpenChunkWriter(ctx, fileName, src) + require.NoError(t, err) + chunkSize = int(info.ChunkSize) + err = writer.Abort(ctx) + require.NoError(t, err) + } + return chunkSize +} + +func TestMultithreadCopy(t *testing.T) { + r := fstest.NewRun(t) + ctx := context.Background() + chunkSize := skipIfNotMultithread(ctx, t, r) + // Check every other transfer for metadata + checkMetadata := false + ctx, ci := fs.AddConfig(ctx) + + for _, upload := range []bool{false, true} { + for _, test := range []struct { + size int + streams int + }{ + {size: chunkSize*2 - 1, streams: 2}, + {size: chunkSize * 2, streams: 2}, + {size: chunkSize*2 + 1, streams: 2}, + } { + checkMetadata = !checkMetadata + ci.Metadata = checkMetadata + fileName := fmt.Sprintf("test-multithread-copy-%v-%d-%d", upload, test.size, test.streams) + t.Run(fmt.Sprintf("upload=%v,size=%v,streams=%v", upload, test.size, test.streams), func(t *testing.T) { + if *fstest.SizeLimit > 0 && int64(test.size) > *fstest.SizeLimit { + t.Skipf("exceeded file size limit %d > %d", test.size, *fstest.SizeLimit) + } + var ( + contents = random.String(test.size) + t1 = fstest.Time("2001-02-03T04:05:06.499999999Z") + file1 fstest.Item + src, dst fs.Object + err error + testMetadata = fs.Metadata{ + // System metadata supported by all backends + "mtime": t1.Format(time.RFC3339Nano), + // User metadata + "potato": "jersey", + } + ) + + var fSrc, fDst fs.Fs + if upload { + file1 = r.WriteFile(fileName, contents, t1) + r.CheckRemoteItems(t) + r.CheckLocalItems(t, file1) + fDst, fSrc = r.Fremote, r.Flocal + } else { + file1 = r.WriteObject(ctx, fileName, contents, t1) + r.CheckRemoteItems(t, file1) + r.CheckLocalItems(t) + fDst, fSrc = r.Flocal, r.Fremote + } + src, err = fSrc.NewObject(ctx, fileName) + require.NoError(t, err) + + do, canSetMetadata := src.(fs.SetMetadataer) + if checkMetadata && canSetMetadata { + // Set metadata on the source if required + err := do.SetMetadata(ctx, testMetadata) + if err == fs.ErrorNotImplemented { + canSetMetadata = false + } else { + require.NoError(t, err) + fstest.CheckEntryMetadata(ctx, t, r.Flocal, src, testMetadata) + } + } + + accounting.GlobalStats().ResetCounters() + tr := accounting.GlobalStats().NewTransfer(src, nil) + + defer func() { + tr.Done(ctx, err) + }() + + dst, err = multiThreadCopy(ctx, fDst, fileName, src, test.streams, tr) + require.NoError(t, err) + + assert.Equal(t, src.Size(), dst.Size()) + assert.Equal(t, fileName, dst.Remote()) + fstest.CheckListingWithPrecision(t, fSrc, []fstest.Item{file1}, nil, fs.GetModifyWindow(ctx, fDst, fSrc)) + fstest.CheckListingWithPrecision(t, fDst, []fstest.Item{file1}, nil, fs.GetModifyWindow(ctx, fDst, fSrc)) + + if checkMetadata && canSetMetadata && fDst.Features().ReadMetadata { + fstest.CheckEntryMetadata(ctx, t, fDst, dst, testMetadata) + } + + require.NoError(t, dst.Remove(ctx)) + require.NoError(t, src.Remove(ctx)) + + }) + } + } +} + +type errorObject struct { + fs.Object + size int64 + wg *sync.WaitGroup +} + +// Open opens the file for read. Call Close() on the returned io.ReadCloser +// +// Remember this is called multiple times whenever the backend seeks (eg having read checksum) +func (o errorObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { + fs.Debugf(nil, "Open with options = %v", options) + rc, err := o.Object.Open(ctx, options...) + if err != nil { + return nil, err + } + // Return an error reader for the second segment + for _, option := range options { + if ropt, ok := option.(*fs.RangeOption); ok { + end := ropt.End + 1 + if end >= o.size { + // Give the other chunks a chance to start + time.Sleep(time.Second) + // Wait for chunks to upload first + o.wg.Wait() + fs.Debugf(nil, "Returning error reader") + return errorReadCloser{rc}, nil + } + } + } + o.wg.Add(1) + return wgReadCloser{rc, o.wg}, nil +} + +type errorReadCloser struct { + io.ReadCloser +} + +func (rc errorReadCloser) Read(p []byte) (n int, err error) { + fs.Debugf(nil, "BOOM: simulated read failure") + return 0, errors.New("BOOM: simulated read failure") +} + +type wgReadCloser struct { + io.ReadCloser + wg *sync.WaitGroup +} + +func (rc wgReadCloser) Close() (err error) { + rc.wg.Done() + return rc.ReadCloser.Close() +} + +// Make sure aborting the multi-thread copy doesn't overwrite an existing file. +func TestMultithreadCopyAbort(t *testing.T) { + r := fstest.NewRun(t) + ctx := context.Background() + chunkSize := skipIfNotMultithread(ctx, t, r) + size := 2*chunkSize + 1 + + if *fstest.SizeLimit > 0 && int64(size) > *fstest.SizeLimit { + t.Skipf("exceeded file size limit %d > %d", size, *fstest.SizeLimit) + } + + // first write a canary file which we are trying not to overwrite + const fileName = "test-multithread-abort" + contents := random.String(100) + t1 := fstest.Time("2001-02-03T04:05:06.499999999Z") + canary := r.WriteObject(ctx, fileName, contents, t1) + r.CheckRemoteItems(t, canary) + + // Now write a local file to upload + file1 := r.WriteFile(fileName, random.String(size), t1) + r.CheckLocalItems(t, file1) + + src, err := r.Flocal.NewObject(ctx, fileName) + require.NoError(t, err) + accounting.GlobalStats().ResetCounters() + tr := accounting.GlobalStats().NewTransfer(src, nil) + + defer func() { + tr.Done(ctx, err) + }() + wg := new(sync.WaitGroup) + dst, err := multiThreadCopy(ctx, r.Fremote, fileName, errorObject{src, int64(size), wg}, 1, tr) + assert.Error(t, err) + assert.Nil(t, dst) + + if r.Fremote.Features().PartialUploads { + r.CheckRemoteItems(t) + + } else { + r.CheckRemoteItems(t, canary) + o, err := r.Fremote.NewObject(ctx, fileName) + require.NoError(t, err) + require.NoError(t, o.Remove(ctx)) + } +} diff --git a/fs/operations/operations.go b/fs/operations/operations.go new file mode 100644 index 0000000..1ada8f8 --- /dev/null +++ b/fs/operations/operations.go @@ -0,0 +1,2742 @@ +// Package operations does generic operations on filesystems and objects +package operations + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/csv" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "os" + "path" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/object" + "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/lib/atexit" + "github.com/rclone/rclone/lib/errcount" + "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/random" + "github.com/rclone/rclone/lib/readers" + "github.com/rclone/rclone/lib/transform" + "golang.org/x/sync/errgroup" + "golang.org/x/text/unicode/norm" +) + +// CheckHashes checks the two files to see if they have common +// known hash types and compares them +// +// Returns. +// +// equal - which is equality of the hashes +// +// hash - the HashType. This is HashNone if either of the hashes were +// unset or a compatible hash couldn't be found. +// +// err - may return an error which will already have been logged +// +// If an error is returned it will return equal as false +func CheckHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object) (equal bool, ht hash.Type, err error) { + common := src.Fs().Hashes().Overlap(dst.Fs().Hashes()) + // fs.Debugf(nil, "Shared hashes: %v", common) + if common.Count() == 0 { + return true, hash.None, nil + } + equal, ht, _, _, err = checkHashes(ctx, src, dst, common.GetOne()) + return equal, ht, err +} + +var errNoHash = errors.New("no hash available") + +// checkHashes does the work of CheckHashes but takes a hash.Type and +// returns the effective hash type used. +func checkHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object, ht hash.Type) (equal bool, htOut hash.Type, srcHash, dstHash string, err error) { + // Calculate hashes in parallel + g, ctx := errgroup.WithContext(ctx) + var srcErr, dstErr error + g.Go(func() (err error) { + srcHash, srcErr = src.Hash(ctx, ht) + if srcErr != nil { + return srcErr + } + if srcHash == "" { + fs.Debugf(src, "Src hash empty - aborting Dst hash check") + return errNoHash + } + return nil + }) + g.Go(func() (err error) { + dstHash, dstErr = dst.Hash(ctx, ht) + if dstErr != nil { + return dstErr + } + if dstHash == "" { + fs.Debugf(dst, "Dst hash empty - aborting Src hash check") + return errNoHash + } + return nil + }) + err = g.Wait() + if err == errNoHash { + return true, hash.None, srcHash, dstHash, nil + } + if srcErr != nil { + err = fs.CountError(ctx, srcErr) + fs.Errorf(src, "Failed to calculate src hash: %v", err) + } + if dstErr != nil { + err = fs.CountError(ctx, dstErr) + fs.Errorf(dst, "Failed to calculate dst hash: %v", err) + } + if err != nil { + return false, ht, srcHash, dstHash, err + } + if srcHash != dstHash { + fs.Debugf(src, "%v = %s (%v)", ht, srcHash, src.Fs()) + fs.Debugf(dst, "%v = %s (%v)", ht, dstHash, dst.Fs()) + return false, ht, srcHash, dstHash, nil + } + fs.Debugf(src, "%v = %s OK", ht, srcHash) + return true, ht, srcHash, dstHash, nil +} + +// Equal checks to see if the src and dst objects are equal by looking at +// size, mtime and hash +// +// If the src and dst size are different then it is considered to be +// not equal. If --size-only is in effect then this is the only check +// that is done. If --ignore-size is in effect then this check is +// skipped and the files are considered the same size. +// +// If the size is the same and the mtime is the same then it is +// considered to be equal. This check is skipped if using --checksum. +// +// If the size is the same and mtime is different, unreadable or +// --checksum is set and the hash is the same then the file is +// considered to be equal. In this case the mtime on the dst is +// updated if --checksum is not set. +// +// Otherwise the file is considered to be not equal including if there +// were errors reading info. +func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool { + return equal(ctx, src, dst, defaultEqualOpt(ctx)) +} + +// DirsEqual is like Equal but for dirs instead of objects. +// It returns true if two dirs should be considered "equal" for the purposes of syncCopyMove +// (in other words, true == "skip updating modtime/metadata for this dir".) +// Unlike Equal, it does not consider size or checksum, as these do not apply to directories. +func DirsEqual(ctx context.Context, src, dst fs.Directory, opt DirsEqualOpt) (equal bool) { + if dst == nil { + return false + } + ci := fs.GetConfig(ctx) + if ci.SizeOnly || ci.Immutable || ci.IgnoreExisting || opt.ModifyWindow == fs.ModTimeNotSupported { + return true + } + if ci.IgnoreTimes { + return false + } + if !(opt.SetDirModtime || opt.SetDirMetadata) { + return true + } + srcModTime, dstModTime := src.ModTime(ctx), dst.ModTime(ctx) + if srcModTime.IsZero() || dstModTime.IsZero() { + return false + } + dt := dstModTime.Sub(srcModTime) + if dt < opt.ModifyWindow && dt > -opt.ModifyWindow { + fs.Debugf(dst, "Directory modification time the same (differ by %s, within tolerance %s)", dt, opt.ModifyWindow) + return true + } + if ci.UpdateOlder && dt >= opt.ModifyWindow { + fs.Debugf(dst, "Destination directory is newer than source, skipping") + return true + } + return false +} + +// sizeDiffers compare the size of src and dst taking into account the +// various ways of ignoring sizes +func sizeDiffers(ctx context.Context, src, dst fs.ObjectInfo) bool { + ci := fs.GetConfig(ctx) + if ci.IgnoreSize || src.Size() < 0 || dst.Size() < 0 { + return false + } + if src.Size() == dst.Size() { + fs.Debugf(dst, "size = %d OK", dst.Size()) + return false + } + fs.Debugf(src, "size = %d (%v)", src.Size(), src.Fs()) + fs.Debugf(dst, "size = %d (%v)", dst.Size(), dst.Fs()) + return true +} + +var checksumWarning sync.Once + +// options for equal function() +type equalOpt struct { + sizeOnly bool // if set only check size + checkSum bool // if set check checksum+size instead of modtime+size + updateModTime bool // if set update the modtime if hashes identical and checking with modtime+size + forceModTimeMatch bool // if set assume modtimes match +} + +// default set of options for equal() +func defaultEqualOpt(ctx context.Context) equalOpt { + ci := fs.GetConfig(ctx) + return equalOpt{ + sizeOnly: ci.SizeOnly, + checkSum: ci.CheckSum, + updateModTime: !ci.NoUpdateModTime, + forceModTimeMatch: false, + } +} + +// DirsEqualOpt represents options for DirsEqual function() +type DirsEqualOpt struct { + ModifyWindow time.Duration // Max time diff to be considered the same + SetDirModtime bool // whether to consider dir modtime + SetDirMetadata bool // whether to consider dir metadata +} + +var modTimeUploadOnce sync.Once + +// emit a log if we are about to upload a file to set its modification time +func logModTimeUpload(dst fs.Object) { + modTimeUploadOnce.Do(func() { + fs.Logf(dst.Fs(), "Forced to upload files to set modification times on this backend.") + }) +} + +// EqualFn allows replacing Equal() with a custom function during NeedTransfer() +type ( + EqualFn func(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool + equalFnContextKey struct{} +) + +var equalFnKey = equalFnContextKey{} + +// WithEqualFn stores equalFn in ctx and returns a copy of ctx in which equalFnKey = equalFn +func WithEqualFn(ctx context.Context, equalFn EqualFn) context.Context { + return context.WithValue(ctx, equalFnKey, equalFn) +} + +func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt) bool { + ci := fs.GetConfig(ctx) + logger, _ := GetLogger(ctx) + if sizeDiffers(ctx, src, dst) { + fs.Debug(src, "Sizes differ") + logger(ctx, Differ, src, dst, nil) + return false + } + if opt.sizeOnly { + fs.Debugf(src, "Sizes identical") + logger(ctx, Match, src, dst, nil) + return true + } + + // Assert: Size is equal or being ignored + + // If checking checksum and not modtime + if opt.checkSum { + // Check the hash + same, ht, _ := CheckHashes(ctx, src, dst) + if !same { + fs.Debugf(src, "%v differ", ht) + logger(ctx, Differ, src, dst, nil) + return false + } + if ht == hash.None { + common := src.Fs().Hashes().Overlap(dst.Fs().Hashes()) + if common.Count() == 0 { + checksumWarning.Do(func() { + fs.Logf(dst.Fs(), "--checksum is in use but the source and destination have no hashes in common; falling back to --size-only") + }) + } + fs.Debugf(src, "Size of src and dst objects identical") + } else { + fs.Debugf(src, "Size and %v of src and dst objects identical", ht) + } + logger(ctx, Match, src, dst, nil) + return true + } + + srcModTime := src.ModTime(ctx) + if !opt.forceModTimeMatch { + // Sizes the same so check the mtime + modifyWindow := fs.GetModifyWindow(ctx, src.Fs(), dst.Fs()) + if modifyWindow == fs.ModTimeNotSupported { + fs.Debugf(src, "Sizes identical") + logger(ctx, Match, src, dst, nil) + return true + } + dstModTime := dst.ModTime(ctx) + dt := dstModTime.Sub(srcModTime) + if dt < modifyWindow && dt > -modifyWindow { + fs.Debugf(src, "Size and modification time the same (differ by %s, within tolerance %s)", dt, modifyWindow) + logger(ctx, Match, src, dst, nil) + return true + } + + fs.Debugf(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime) + } + + // Check if the hashes are the same + same, ht, _ := CheckHashes(ctx, src, dst) + if !same { + fs.Debugf(src, "%v differ", ht) + logger(ctx, Differ, src, dst, nil) + return false + } + if ht == hash.None && !ci.RefreshTimes { + // if couldn't check hash, return that they differ + logger(ctx, Differ, src, dst, nil) + return false + } + + // mod time differs but hash is the same to reset mod time if required + if opt.updateModTime { + if !SkipDestructive(ctx, src, "update modification time") { + // Size and hash the same but mtime different + // Error if objects are treated as immutable + if ci.Immutable { + fs.Errorf(dst, "Timestamp mismatch between immutable objects") + logger(ctx, Differ, src, dst, nil) + return false + } + // Update the mtime of the dst object here + err := dst.SetModTime(ctx, srcModTime) + if errors.Is(err, fs.ErrorCantSetModTime) { + logModTimeUpload(dst) + fs.Infof(dst, "src and dst identical but can't set mod time without re-uploading") + logger(ctx, Differ, src, dst, nil) + return false + } else if errors.Is(err, fs.ErrorCantSetModTimeWithoutDelete) { + logModTimeUpload(dst) + fs.Infof(dst, "src and dst identical but can't set mod time without deleting and re-uploading") + // Remove the file if BackupDir isn't set. If BackupDir is set we would rather have the old file + // put in the BackupDir than deleted which is what will happen if we don't delete it. + if ci.BackupDir == "" { + err = dst.Remove(ctx) + if err != nil { + fs.Errorf(dst, "failed to delete before re-upload: %v", err) + } + } + logger(ctx, Differ, src, dst, nil) + return false + } else if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(dst, "Failed to set modification time: %v", err) + } else { + fs.Infof(src, "Updated modification time in destination") + } + } + } + logger(ctx, Match, src, dst, nil) + return true +} + +// CommonHash returns a single hash.Type and a HashOption with that +// type which is in common between the two fs.Fs. +func CommonHash(ctx context.Context, fa, fb fs.Info) (hash.Type, *fs.HashesOption) { + ci := fs.GetConfig(ctx) + // work out which hash to use - limit to 1 hash in common + var common hash.Set + hashType := hash.None + if !ci.IgnoreChecksum { + common = fb.Hashes().Overlap(fa.Hashes()) + if common.Count() > 0 { + hashType = common.GetOne() + common = hash.Set(hashType) + } + } + return hashType, &fs.HashesOption{Hashes: common} +} + +// SameObject returns true if src and dst could be pointing to the +// same object. +func SameObject(src, dst fs.Object) bool { + srcFs, dstFs := src.Fs(), dst.Fs() + if !SameConfig(srcFs, dstFs) { + // If same remote type then check ID of objects if available + doSrcID, srcIDOK := src.(fs.IDer) + doDstID, dstIDOK := dst.(fs.IDer) + if srcIDOK && dstIDOK && SameRemoteType(srcFs, dstFs) { + srcID, dstID := doSrcID.ID(), doDstID.ID() + if srcID != "" && srcID == dstID { + return true + } + } + return false + } + srcPath := path.Join(srcFs.Root(), src.Remote()) + dstPath := path.Join(dstFs.Root(), dst.Remote()) + if srcFs.Features().IsLocal && dstFs.Features().IsLocal && runtime.GOOS == "darwin" { + if norm.NFC.String(srcPath) == norm.NFC.String(dstPath) { + return true + } + } + if dst.Fs().Features().CaseInsensitive { + srcPath = strings.ToLower(srcPath) + dstPath = strings.ToLower(dstPath) + } + return srcPath == dstPath +} + +// Move src object to dst or fdst if nil. If dst is nil then it uses +// remote as the name of the new object. +// +// Note that you must check the destination does not exist before +// calling this and pass it as dst. If you pass dst=nil and the +// destination does exist then this may create duplicates or return +// errors. +// +// It returns the destination object if possible. Note that this may +// be nil. +// +// This is accounted as a check. +func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) { + return move(ctx, fdst, dst, remote, src, false) +} + +// MoveTransfer moves src object to dst or fdst if nil. If dst is nil +// then it uses remote as the name of the new object. +// +// This is identical to Move but is accounted as a transfer. +func MoveTransfer(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) { + return move(ctx, fdst, dst, remote, src, true) +} + +// move - see Move for help +func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object, isTransfer bool) (newDst fs.Object, err error) { + origRemote := remote // avoid double-transform on fallback to copy + remote = transform.Path(ctx, remote, false) + ci := fs.GetConfig(ctx) + newDst = dst + if ci.DryRun && dst != nil && SameObject(src, dst) && src.Remote() == transform.Path(ctx, dst.Remote(), false) { + return // avoid SkipDestructive log for objects that won't really be moved + } + var tr *accounting.Transfer + if isTransfer { + tr = accounting.Stats(ctx).NewTransfer(src, fdst) + } else { + tr = accounting.Stats(ctx).NewCheckingTransfer(src, "moving") + } + defer func() { + if err == nil { + accounting.Stats(ctx).Renames(1) + } + tr.Done(ctx, err) + }() + action := "move" + if remote != src.Remote() { + action += " to " + remote + } + if SkipDestructive(ctx, src, action) { + in := tr.Account(ctx, nil) + in.DryRun(src.Size()) + return newDst, nil + } + // See if we have Move available + if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && (fdst.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) { + // Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive) + if dst != nil { + remote = transform.Path(ctx, dst.Remote(), false) + if !SameObject(src, dst) { + err = DeleteFile(ctx, dst) + if err != nil { + return newDst, err + } + } else if src.Remote() == remote { + return newDst, nil + } else if needsMoveCaseInsensitive(fdst, fdst, remote, src.Remote(), false) { + doMove = func(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + return MoveCaseInsensitive(ctx, fdst, fdst, remote, src.Remote(), false, src) + } + } + } else if needsMoveCaseInsensitive(fdst, fdst, remote, src.Remote(), false) { + doMove = func(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + return MoveCaseInsensitive(ctx, fdst, fdst, remote, src.Remote(), false, src) + } + } + // Move dst <- src + in := tr.Account(ctx, nil) // account the transfer + in.ServerSideTransferStart() + newDst, err = doMove(ctx, src, remote) + switch err { + case nil: + if newDst != nil && src.String() != newDst.String() { + fs.Infof(src, "Moved (server-side) to: %s", newDst.String()) + } else { + fs.Infof(src, "Moved (server-side)") + } + in.ServerSideMoveEnd(newDst.Size()) // account the bytes for the server-side transfer + _ = in.Close() + return newDst, nil + case fs.ErrorCantMove: + fs.Debugf(src, "Can't move, switching to copy") + _ = in.Close() + default: + err = fs.CountError(ctx, err) + fs.Errorf(src, "Couldn't move: %v", err) + _ = in.Close() + return newDst, err + } + } + // Move not found or didn't work so copy dst <- src + if origRemote != remote { + dst = nil + } + newDst, err = Copy(ctx, fdst, dst, origRemote, src) + if err != nil { + fs.Errorf(src, "Not deleting source as copy failed: %v", err) + return newDst, err + } + // Delete src if no error on copy + return newDst, DeleteFile(ctx, src) +} + +// CanServerSideMove returns true if fdst support server-side moves or +// server-side copies +// +// Some remotes simulate rename by server-side copy and delete, so include +// remotes that implements either Mover or Copier. +func CanServerSideMove(fdst fs.Fs) bool { + canMove := fdst.Features().Move != nil + canCopy := fdst.Features().Copy != nil + return canMove || canCopy +} + +// SuffixName adds the current --suffix to the remote, obeying +// --suffix-keep-extension if set +func SuffixName(ctx context.Context, remote string) string { + ci := fs.GetConfig(ctx) + if ci.Suffix == "" { + return remote + } + if ci.SuffixKeepExtension { + return transform.SuffixKeepExtension(remote, ci.Suffix) + } + return remote + ci.Suffix +} + +// DeleteFileWithBackupDir deletes a single file respecting --dry-run +// and accumulating stats and errors. +// +// If backupDir is set then it moves the file to there instead of +// deleting +func DeleteFileWithBackupDir(ctx context.Context, dst fs.Object, backupDir fs.Fs) (err error) { + tr := accounting.Stats(ctx).NewCheckingTransfer(dst, "deleting") + defer func() { + tr.Done(ctx, err) + }() + err = accounting.Stats(ctx).DeleteFile(ctx, dst.Size()) + if err != nil { + return err + } + action, actioned := "delete", "Deleted" + if backupDir != nil { + action, actioned = "move into backup dir", "Moved into backup dir" + } + skip := SkipDestructive(ctx, dst, action) + if skip { + // do nothing + } else if backupDir != nil { + err = MoveBackupDir(ctx, backupDir, dst) + } else { + err = dst.Remove(ctx) + } + if err != nil { + fs.Errorf(dst, "Couldn't %s: %v", action, err) + err = fs.CountError(ctx, err) + } else if !skip { + fs.Infof(dst, "%s", actioned) + } + return err +} + +// DeleteFile deletes a single file respecting --dry-run and accumulating stats and errors. +// +// If useBackupDir is set and --backup-dir is in effect then it moves +// the file to there instead of deleting +func DeleteFile(ctx context.Context, dst fs.Object) (err error) { + return DeleteFileWithBackupDir(ctx, dst, nil) +} + +// DeleteFilesWithBackupDir removes all the files passed in the +// channel +// +// If backupDir is set the files will be placed into that directory +// instead of being deleted. +func DeleteFilesWithBackupDir(ctx context.Context, toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error { + var wg sync.WaitGroup + ci := fs.GetConfig(ctx) + wg.Add(ci.Checkers) + var errorCount atomic.Int32 + var fatalErrorCount atomic.Int32 + + for range ci.Checkers { + go func() { + defer wg.Done() + for dst := range toBeDeleted { + err := DeleteFileWithBackupDir(ctx, dst, backupDir) + if err != nil { + errorCount.Add(1) + logger, _ := GetLogger(ctx) + logger(ctx, TransferError, nil, dst, err) + if fserrors.IsFatalError(err) { + fs.Errorf(dst, "Got fatal error on delete: %s", err) + fatalErrorCount.Add(1) + return + } + } + } + }() + } + fs.Debugf(nil, "Waiting for deletions to finish") + wg.Wait() + if errorCount.Load() > 0 { + err := fmt.Errorf("failed to delete %d files", errorCount.Load()) + if fatalErrorCount.Load() > 0 { + return fserrors.FatalError(err) + } + return err + } + return nil +} + +// DeleteFiles removes all the files passed in the channel +func DeleteFiles(ctx context.Context, toBeDeleted fs.ObjectsChan) error { + return DeleteFilesWithBackupDir(ctx, toBeDeleted, nil) +} + +// ReadFile reads the object into memory and accounts it +func ReadFile(ctx context.Context, o fs.Object) (b []byte, err error) { + tr := accounting.Stats(ctx).NewTransfer(o, nil) + defer func() { + tr.Done(ctx, err) + }() + in0, err := Open(ctx, o) + if err != nil { + return nil, fmt.Errorf("failed to open %v: %w", o, err) + } + in := tr.Account(ctx, in0).WithBuffer() // account and buffer the transfer + defer fs.CheckClose(in, &err) // closes in0 also + b, err = io.ReadAll(in) + if err != nil { + return nil, fmt.Errorf("failed to read %v: %w", o, err) + } + return b, nil +} + +// SameRemoteType returns true if fdst and fsrc are the same type +func SameRemoteType(fdst, fsrc fs.Info) bool { + return fmt.Sprintf("%T", fdst) == fmt.Sprintf("%T", fsrc) +} + +// SameConfig returns true if fdst and fsrc are using the same config +// file entry +func SameConfig(fdst, fsrc fs.Info) bool { + return fdst.Name() == fsrc.Name() +} + +// SameConfigArr returns true if any of []fsrcs has same config file entry with fdst +func SameConfigArr(fdst fs.Info, fsrcs []fs.Fs) bool { + for _, fsrc := range fsrcs { + if fdst.Name() == fsrc.Name() { + return true + } + } + return false +} + +// Same returns true if fdst and fsrc point to the same underlying Fs +func Same(fdst, fsrc fs.Info) bool { + return SameConfig(fdst, fsrc) && strings.Trim(fdst.Root(), "/") == strings.Trim(fsrc.Root(), "/") +} + +// fixRoot returns the Root with a trailing / if not empty. +// +// It returns a case folded version for case insensitive file systems +func fixRoot(f fs.Info) (s string, folded string) { + s = strings.Trim(filepath.ToSlash(f.Root()), "/") + if s != "" { + s += "/" + } + folded = s + if f.Features().CaseInsensitive { + folded = strings.ToLower(s) + } + return s, folded +} + +// OverlappingFilterCheck returns true if fdst and fsrc point to the same +// underlying Fs and they overlap without fdst being excluded by any filter rule. +func OverlappingFilterCheck(ctx context.Context, fdst fs.Fs, fsrc fs.Fs) bool { + if !SameConfig(fdst, fsrc) { + return false + } + fdstRoot, fdstRootFolded := fixRoot(fdst) + fsrcRoot, fsrcRootFolded := fixRoot(fsrc) + if fdstRootFolded == fsrcRootFolded { + return true + } else if strings.HasPrefix(fdstRootFolded, fsrcRootFolded) { + fdstRelative := fdstRoot[len(fsrcRoot):] + return filterCheck(ctx, fsrc, fdstRelative) + } else if strings.HasPrefix(fsrcRootFolded, fdstRootFolded) { + fsrcRelative := fsrcRoot[len(fdstRoot):] + return filterCheck(ctx, fdst, fsrcRelative) + } + return false +} + +// filterCheck checks if dir is included in f +func filterCheck(ctx context.Context, f fs.Fs, dir string) bool { + fi := filter.GetConfig(ctx) + includeDirectory := fi.IncludeDirectory(ctx, f) + include, err := includeDirectory(dir) + if err != nil { + fs.Errorf(f, "Failed to discover whether directory is included: %v", err) + return true + } + return include +} + +// SameDir returns true if fdst and fsrc point to the same +// underlying Fs and they are the same directory. +func SameDir(fdst, fsrc fs.Info) bool { + if !SameConfig(fdst, fsrc) { + return false + } + _, fdstRootFolded := fixRoot(fdst) + _, fsrcRootFolded := fixRoot(fsrc) + return fdstRootFolded == fsrcRootFolded +} + +// Retry runs fn up to maxTries times if it returns a retriable error +func Retry(ctx context.Context, o any, maxTries int, fn func() error) (err error) { + for tries := 1; tries <= maxTries; tries++ { + // Call the function which might error + err = fn() + if err == nil { + break + } + // End if ctx is in error + if fserrors.ContextError(ctx, &err) { + break + } + // Retry if err returned a retry error + if fserrors.IsRetryError(err) || fserrors.ShouldRetry(err) { + fs.Debugf(o, "Received error: %v - low level retry %d/%d", err, tries, maxTries) + continue + } else if t, ok := pacer.IsRetryAfter(err); ok { + fs.Debugf(o, "Sleeping for %v (as indicated by the server) to obey Retry-After error: %v", t, err) + time.Sleep(t) + continue + } + break + } + return err +} + +// ListFn lists the Fs to the supplied function +// +// Lists in parallel which may get them out of order +func ListFn(ctx context.Context, f fs.Fs, fn func(fs.Object)) error { + ci := fs.GetConfig(ctx) + return walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error { + entries.ForObject(fn) + return nil + }) +} + +// StdoutMutex mutex for synchronized output on stdout +var StdoutMutex sync.Mutex + +// SyncPrintf is a global var holding the Printf function so that it +// can be overridden. +// +// This writes to stdout holding the StdoutMutex. If you are going to +// override it and write to os.Stdout then you should hold the +// StdoutMutex too. +var SyncPrintf = func(format string, a ...any) { + StdoutMutex.Lock() + defer StdoutMutex.Unlock() + fmt.Printf(format, a...) +} + +// SyncFprintf - Synchronized fmt.Fprintf +// +// Ignores errors from Fprintf. +// +// Prints to stdout if w is nil +func SyncFprintf(w io.Writer, format string, a ...any) { + if w == nil || w == os.Stdout { + SyncPrintf(format, a...) + } else { + StdoutMutex.Lock() + defer StdoutMutex.Unlock() + _, _ = fmt.Fprintf(w, format, a...) + } +} + +// SizeString make string representation of size for output +// +// Optional human-readable format including a binary suffix +func SizeString(size int64, humanReadable bool) string { + if humanReadable { + if size < 0 { + return "-" + fs.SizeSuffix(-size).String() + } + return fs.SizeSuffix(size).String() + } + return strconv.FormatInt(size, 10) +} + +// SizeStringField make string representation of size for output in fixed width field +// +// Optional human-readable format including a binary suffix +// Argument rawWidth is used to format field with of raw value. When humanReadable +// option the width is hard coded to 9, since SizeSuffix strings have precision 3 +// and longest value will be "999.999Ei". This way the width can be optimized +// depending to the humanReadable option. To always use a longer width the return +// value can always be fed into another format string with a specific field with. +func SizeStringField(size int64, humanReadable bool, rawWidth int) string { + str := SizeString(size, humanReadable) + if humanReadable { + return fmt.Sprintf("%9s", str) + } + return fmt.Sprintf("%[2]*[1]s", str, rawWidth) +} + +// CountString make string representation of count for output +// +// Optional human-readable format including a decimal suffix +func CountString(count int64, humanReadable bool) string { + if humanReadable { + if count < 0 { + return "-" + fs.CountSuffix(-count).String() + } + return fs.CountSuffix(count).String() + } + return strconv.FormatInt(count, 10) +} + +// CountStringField make string representation of count for output in fixed width field +// +// Similar to SizeStringField, but human readable with decimal prefix and field width 8 +// since there is no 'i' in the decimal prefix symbols (e.g. "999.999E") +func CountStringField(count int64, humanReadable bool, rawWidth int) string { + str := CountString(count, humanReadable) + if humanReadable { + return fmt.Sprintf("%8s", str) + } + return fmt.Sprintf("%[2]*[1]s", str, rawWidth) +} + +// List the Fs to the supplied writer +// +// Shows size and path - obeys includes and excludes. +// +// Lists in parallel which may get them out of order +func List(ctx context.Context, f fs.Fs, w io.Writer) error { + ci := fs.GetConfig(ctx) + return ListFn(ctx, f, func(o fs.Object) { + SyncFprintf(w, "%s %s\n", SizeStringField(o.Size(), ci.HumanReadable, 9), o.Remote()) + }) +} + +// ListLong lists the Fs to the supplied writer +// +// Shows size, mod time and path - obeys includes and excludes. +// +// Lists in parallel which may get them out of order +func ListLong(ctx context.Context, f fs.Fs, w io.Writer) error { + ci := fs.GetConfig(ctx) + return ListFn(ctx, f, func(o fs.Object) { + tr := accounting.Stats(ctx).NewCheckingTransfer(o, "listing") + defer func() { + tr.Done(ctx, nil) + }() + modTime := o.ModTime(ctx) + SyncFprintf(w, "%s %s %s\n", SizeStringField(o.Size(), ci.HumanReadable, 9), modTime.Local().Format("2006-01-02 15:04:05.000000000"), o.Remote()) + }) +} + +// HashSum returns the human-readable hash for ht passed in. This may +// be UNSUPPORTED or ERROR. If it isn't returning a valid hash it will +// return an error. +func HashSum(ctx context.Context, ht hash.Type, base64Encoded bool, downloadFlag bool, o fs.Object) (string, error) { + var sum string + var err error + + // If downloadFlag is true, download and hash the file. + // If downloadFlag is false, call o.Hash asking the remote for the hash + if downloadFlag { + // Setup: Define accounting, open the file with NewReOpen to provide restarts, account for the transfer, and setup a multi-hasher with the appropriate type + // Execution: io.Copy file to hasher, get hash and encode in hex + + tr := accounting.Stats(ctx).NewTransfer(o, nil) + defer func() { + tr.Done(ctx, err) + }() + + // Open with NewReOpen to provide restarts + var options []fs.OpenOption + for _, option := range fs.GetConfig(ctx).DownloadHeaders { + options = append(options, option) + } + var in io.ReadCloser + in, err = Open(ctx, o, options...) + if err != nil { + return "ERROR", fmt.Errorf("failed to open file %v: %w", o, err) + } + + // Account and buffer the transfer + in = tr.Account(ctx, in).WithBuffer() + + // Setup hasher + hasher, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht)) + if err != nil { + return "UNSUPPORTED", fmt.Errorf("hash unsupported: %w", err) + } + + // Copy to hasher, downloading the file and passing directly to hash + _, err = io.Copy(hasher, in) + if err != nil { + return "ERROR", fmt.Errorf("failed to copy file to hasher: %w", err) + } + + // Get hash as hex or base64 encoded string + sum, err = hasher.SumString(ht, base64Encoded) + if err != nil { + return "ERROR", fmt.Errorf("hasher returned an error: %w", err) + } + } else { + tr := accounting.Stats(ctx).NewCheckingTransfer(o, "hashing") + defer func() { + tr.Done(ctx, err) + }() + + sum, err = o.Hash(ctx, ht) + if base64Encoded { + hexBytes, _ := hex.DecodeString(sum) + sum = base64.URLEncoding.EncodeToString(hexBytes) + } + if err == hash.ErrUnsupported { + return "", fmt.Errorf("hash unsupported: %w", err) + } + if err != nil { + return "", fmt.Errorf("failed to get hash %v from backend: %w", ht, err) + } + } + + return sum, nil +} + +// HashLister does an md5sum equivalent for the hash type passed in +// Updated to handle both standard hex encoding and base64 +// Updated to perform multiple hashes concurrently +func HashLister(ctx context.Context, ht hash.Type, outputBase64 bool, downloadFlag bool, f fs.Fs, w io.Writer) error { + width := hash.Width(ht, outputBase64) + // Use --checkers concurrency unless downloading in which case use --transfers + concurrency := fs.GetConfig(ctx).Checkers + if downloadFlag { + concurrency = fs.GetConfig(ctx).Transfers + } + concurrencyControl := make(chan struct{}, concurrency) + var wg sync.WaitGroup + err := ListFn(ctx, f, func(o fs.Object) { + wg.Add(1) + concurrencyControl <- struct{}{} + go func() { + defer func() { + <-concurrencyControl + wg.Done() + }() + sum, err := HashSum(ctx, ht, outputBase64, downloadFlag, o) + if err != nil { + fs.Errorf(o, "%v", fs.CountError(ctx, err)) + return + } + SyncFprintf(w, "%*s %s\n", width, sum, o.Remote()) + }() + }) + wg.Wait() + return err +} + +// HashSumStream outputs a line compatible with md5sum to w based on the +// input stream in and the hash type ht passed in. If outputBase64 is +// set then the hash will be base64 instead of hexadecimal. +func HashSumStream(ht hash.Type, outputBase64 bool, in io.ReadCloser, w io.Writer) error { + hasher, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht)) + if err != nil { + return fmt.Errorf("hash unsupported: %w", err) + } + written, err := io.Copy(hasher, in) + fs.Debugf(nil, "Creating %s hash of %d bytes read from input stream", ht, written) + if err != nil { + return fmt.Errorf("failed to copy input to hasher: %w", err) + } + sum, err := hasher.SumString(ht, outputBase64) + if err != nil { + return fmt.Errorf("hasher returned an error: %w", err) + } + width := hash.Width(ht, outputBase64) + SyncFprintf(w, "%*s -\n", width, sum) + return nil +} + +// Count counts the objects and their sizes in the Fs +// +// Obeys includes and excludes +func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, sizelessObjects int64, err error) { + err = ListFn(ctx, f, func(o fs.Object) { + atomic.AddInt64(&objects, 1) + objectSize := o.Size() + if objectSize < 0 { + atomic.AddInt64(&sizelessObjects, 1) + } else if objectSize > 0 { + atomic.AddInt64(&size, objectSize) + } + }) + return +} + +// ConfigMaxDepth returns the depth to use for a recursive or non recursive listing. +func ConfigMaxDepth(ctx context.Context, recursive bool) int { + ci := fs.GetConfig(ctx) + depth := ci.MaxDepth + if !recursive && depth < 0 { + depth = 1 + } + return depth +} + +// ListDir lists the directories/buckets/containers in the Fs to the supplied writer +func ListDir(ctx context.Context, f fs.Fs, w io.Writer) error { + ci := fs.GetConfig(ctx) + return walk.ListR(ctx, f, "", false, ConfigMaxDepth(ctx, false), walk.ListDirs, func(entries fs.DirEntries) error { + entries.ForDir(func(dir fs.Directory) { + if dir != nil { + SyncFprintf(w, "%s %13s %s %s\n", SizeStringField(dir.Size(), ci.HumanReadable, 12), dir.ModTime(ctx).Local().Format("2006-01-02 15:04:05"), CountStringField(dir.Items(), ci.HumanReadable, 9), dir.Remote()) + } + }) + return nil + }) +} + +// Mkdir makes a destination directory or container +func Mkdir(ctx context.Context, f fs.Fs, dir string) error { + if SkipDestructive(ctx, fs.LogDirName(f, dir), "make directory") { + return nil + } + fs.Infof(fs.LogDirName(f, dir), "Making directory") + err := f.Mkdir(ctx, dir) + if err != nil { + err = fs.CountError(ctx, err) + return err + } + return nil +} + +// MkdirMetadata makes a destination directory or container with metadata +// +// If the destination Fs doesn't support this it will fall back to +// Mkdir and in this case newDst will be nil. +func MkdirMetadata(ctx context.Context, f fs.Fs, dir string, metadata fs.Metadata) (newDst fs.Directory, err error) { + do := f.Features().MkdirMetadata + if do == nil { + return nil, Mkdir(ctx, f, dir) + } + logName := fs.LogDirName(f, dir) + if SkipDestructive(ctx, logName, "make directory") { + return nil, nil + } + fs.Debugf(fs.LogDirName(f, dir), "Making directory with metadata") + newDst, err = do(ctx, dir, metadata) + if err != nil { + err = fs.CountError(ctx, err) + return nil, err + } + if mtime, ok := metadata["mtime"]; ok { + fs.Infof(logName, "Made directory with metadata (mtime=%s)", mtime) + } else { + fs.Infof(logName, "Made directory with metadata") + } + return newDst, err +} + +// MkdirModTime makes a destination directory or container with modtime +// +// It will try to make the directory with MkdirMetadata and if that +// succeeds it will return a non-nil newDst. In all other cases newDst +// will be nil. +// +// If the directory was created with MkDir then it will attempt to use +// Fs.DirSetModTime to update the directory modtime if available. +func MkdirModTime(ctx context.Context, f fs.Fs, dir string, modTime time.Time) (newDst fs.Directory, err error) { + logName := fs.LogDirName(f, dir) + if SkipDestructive(ctx, logName, "make directory") { + return nil, nil + } + metadata := fs.Metadata{ + "mtime": modTime.Format(time.RFC3339Nano), + } + newDst, err = MkdirMetadata(ctx, f, dir, metadata) + if err != nil { + return nil, err + } + if newDst != nil { + // The directory was created and we have logged already + return newDst, nil + } + // The directory was created with Mkdir then we should try to set the time + if do := f.Features().DirSetModTime; do != nil { + err = do(ctx, dir, modTime) + } + fs.Infof(logName, "Made directory with modification time %v", modTime) + return newDst, err +} + +// TryRmdir removes a container but not if not empty. It doesn't +// count errors but may return one. +func TryRmdir(ctx context.Context, f fs.Fs, dir string) error { + accounting.Stats(ctx).DeletedDirs(1) + if SkipDestructive(ctx, fs.LogDirName(f, dir), "remove directory") { + return nil + } + fs.Infof(fs.LogDirName(f, dir), "Removing directory") + return f.Rmdir(ctx, dir) +} + +// Rmdir removes a container but not if not empty +func Rmdir(ctx context.Context, f fs.Fs, dir string) error { + err := TryRmdir(ctx, f, dir) + if err != nil { + err = fs.CountError(ctx, err) + return err + } + return err +} + +// Purge removes a directory and all of its contents +func Purge(ctx context.Context, f fs.Fs, dir string) (err error) { + doFallbackPurge := true + if doPurge := f.Features().Purge; doPurge != nil { + doFallbackPurge = false + accounting.Stats(ctx).DeletedDirs(1) + if SkipDestructive(ctx, fs.LogDirName(f, dir), "purge directory") { + return nil + } + err = doPurge(ctx, dir) + if errors.Is(err, fs.ErrorCantPurge) { + doFallbackPurge = true + } + } + if doFallbackPurge { + // DeleteFiles and Rmdir observe --dry-run + err = DeleteFiles(ctx, listToChan(ctx, f, dir)) + if err != nil { + return err + } + err = Rmdirs(ctx, f, dir, false) + } + if err != nil { + err = fs.CountError(ctx, err) + return err + } + return nil +} + +// Delete removes all the contents of a container. Unlike Purge, it +// obeys includes and excludes. +func Delete(ctx context.Context, f fs.Fs) error { + ci := fs.GetConfig(ctx) + delChan := make(fs.ObjectsChan, ci.Checkers) + delErr := make(chan error, 1) + go func() { + delErr <- DeleteFiles(ctx, delChan) + }() + err := ListFn(ctx, f, func(o fs.Object) { + delChan <- o + }) + close(delChan) + delError := <-delErr + if err == nil { + err = delError + } + return err +} + +// RemoveExisting removes an existing file in a safe way so that it +// can be restored if the operation fails. +// +// This first detects if there is an existing file and renames it to a +// temporary name if there is. +// +// The returned cleanup function should be called on a defer statement +// with a pointer to the error returned. It will revert the changes if +// there is an error or delete the existing file if not. +func RemoveExisting(ctx context.Context, f fs.Fs, remote string, operation string) (cleanup func(*error), err error) { + existingObj, err := f.NewObject(ctx, remote) + if err != nil { + return func(*error) {}, nil + } + doMove := f.Features().Move + if doMove == nil { + return nil, fmt.Errorf("%s: destination file exists already and can't rename", operation) + } + + // Avoid making the leaf name longer if it's already lengthy to avoid + // trouble with file name length limits. + suffix := "." + random.String(8) + var remoteSaved string + if len(path.Base(remote)) > 100 { + remoteSaved = TruncateString(remote, len(remote)-len(suffix)) + suffix + } else { + remoteSaved = remote + suffix + } + + fs.Debugf(existingObj, "%s: renaming existing object to %q before starting", operation, remoteSaved) + existingObj, err = doMove(ctx, existingObj, remoteSaved) + if err != nil { + return nil, fmt.Errorf("%s: failed to rename existing file: %w", operation, err) + } + return func(perr *error) { + if *perr == nil { + fs.Debugf(existingObj, "%s: removing renamed existing file after operation", operation) + err := existingObj.Remove(ctx) + if err != nil { + *perr = fmt.Errorf("%s: failed to remove renamed existing file: %w", operation, err) + } + } else { + fs.Debugf(existingObj, "%s: renaming existing back after failed operation", operation) + _, renameErr := doMove(ctx, existingObj, remote) + if renameErr != nil { + fs.Errorf(existingObj, "%s: failed to restore existing file after failed operation: %v", operation, renameErr) + } + } + }, nil +} + +// listToChan will transfer all objects in the listing to the output +// +// If an error occurs, the error will be logged, and it will close the +// channel. +// +// If the error was ErrorDirNotFound then it will be ignored +func listToChan(ctx context.Context, f fs.Fs, dir string) fs.ObjectsChan { + ci := fs.GetConfig(ctx) + o := make(fs.ObjectsChan, ci.Checkers) + go func() { + defer close(o) + err := walk.ListR(ctx, f, dir, true, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error { + entries.ForObject(func(obj fs.Object) { + o <- obj + }) + return nil + }) + if err != nil && err != fs.ErrorDirNotFound { + err = fmt.Errorf("failed to list: %w", err) + err = fs.CountError(ctx, err) + fs.Errorf(nil, "%v", err) + } + }() + return o +} + +// CleanUp removes the trash for the Fs +func CleanUp(ctx context.Context, f fs.Fs) error { + doCleanUp := f.Features().CleanUp + if doCleanUp == nil { + return fmt.Errorf("%v doesn't support cleanup", f) + } + if SkipDestructive(ctx, f, "clean up old files") { + return nil + } + return doCleanUp(ctx) +} + +// wrap a Reader and a Closer together into a ReadCloser +type readCloser struct { + io.Reader + io.Closer +} + +// Cat any files to the io.Writer +// +// if offset == 0 it will be ignored +// if offset > 0 then the file will be seeked to that offset +// if offset < 0 then the file will be seeked that far from the end +// +// if count < 0 then it will be ignored +// if count >= 0 then only that many characters will be output +func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []byte) error { + var mu sync.Mutex + ci := fs.GetConfig(ctx) + return ListFn(ctx, f, func(o fs.Object) { + var err error + tr := accounting.Stats(ctx).NewTransfer(o, nil) + defer func() { + tr.Done(ctx, err) + }() + opt := fs.RangeOption{Start: offset, End: -1} + size := o.Size() + if opt.Start < 0 { + opt.Start += size + } + if count >= 0 { + opt.End = opt.Start + count - 1 + } + var options []fs.OpenOption + if opt.Start > 0 || opt.End >= 0 { + options = append(options, &opt) + } + for _, option := range ci.DownloadHeaders { + options = append(options, option) + } + var in io.ReadCloser + in, err = Open(ctx, o, options...) + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(o, "Failed to open: %v", err) + return + } + if count >= 0 { + in = &readCloser{Reader: &io.LimitedReader{R: in, N: count}, Closer: in} + } + in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer + // take the lock just before we output stuff, so at the last possible moment + mu.Lock() + defer mu.Unlock() + _, err = io.Copy(w, in) + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(o, "Failed to send to output: %v", err) + } + if len(sep) > 0 { + _, err = w.Write(sep) + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(o, "Failed to send separator to output: %v", err) + } + } + }) +} + +// Rcat reads data from the Reader until EOF and uploads it to a file on remote +// +// in is closed at the end of the transfer +func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time, meta fs.Metadata) (dst fs.Object, err error) { + return rcatSrc(ctx, fdst, dstFileName, in, modTime, meta, nil) +} + +// rcatSrc reads data from the Reader until EOF and uploads it to a file on remote +// +// in is closed at the end of the transfer +// +// Pass in fsrc if known or nil if not +func rcatSrc(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time, meta fs.Metadata, fsrc fs.Fs) (dst fs.Object, err error) { + if SkipDestructive(ctx, dstFileName, "upload from pipe") { + // prevents "broken pipe" errors + _, err = io.Copy(io.Discard, in) + return nil, err + } + + ci := fs.GetConfig(ctx) + tr := accounting.Stats(ctx).NewTransferRemoteSize(dstFileName, -1, nil, fdst) + defer func() { + tr.Done(ctx, err) + }() + var streamIn io.Reader = tr.Account(ctx, in).WithBuffer() + + readCounter := readers.NewCountingReader(streamIn) + var trackingIn io.Reader + var hasher *hash.MultiHasher + var options []fs.OpenOption + if !ci.IgnoreChecksum { + hashes := hash.NewHashSet(fdst.Hashes().GetOne()) // just pick one hash + hashOption := &fs.HashesOption{Hashes: hashes} + options = append(options, hashOption) + hasher, err = hash.NewMultiHasherTypes(hashes) + if err != nil { + return nil, err + } + trackingIn = io.TeeReader(readCounter, hasher) + } else { + trackingIn = readCounter + } + for _, option := range ci.UploadHeaders { + options = append(options, option) + } + if ci.MetadataSet != nil { + options = append(options, fs.MetadataOption(ci.MetadataSet)) + } + + // get the sums from the hasher if in use, or nil + getSums := func() (sums map[hash.Type]string) { + if hasher != nil { + sums = hasher.Sums() + } + return sums + } + + // Read the start of the input and check if it is small enough for direct upload + buf := make([]byte, ci.StreamingUploadCutoff) + fileIsSmall := false + if n, err := io.ReadFull(trackingIn, buf); err == io.EOF || err == io.ErrUnexpectedEOF { + fileIsSmall = true + buf = buf[:n] + } + + // Read the data we have already read in buf and any further unread + streamIn = io.MultiReader(bytes.NewReader(buf), trackingIn) + + doPutStream := fdst.Features().PutStream + + // Upload the input + if fileIsSmall || doPutStream == nil { + var rs io.ReadSeeker + if fileIsSmall { + fs.Debugf(fdst, "File to upload is small (%d bytes), uploading instead of streaming", len(buf)) + rs = bytes.NewReader(buf) + } else { + fs.Debugf(fdst, "Target remote doesn't support streaming uploads, creating temporary local FS to spool file") + spool, err := os.CreateTemp("", "rclone-spool") + if err != nil { + return nil, fmt.Errorf("failed to create temporary spool file: %v", err) + } + fileName := spool.Name() + defer func() { + err := spool.Close() + if err != nil { + fs.Errorf(fileName, "Failed to close temporary spool file: %v", err) + } + err = os.Remove(fileName) + if err != nil { + fs.Errorf(fileName, "Failed to delete temporary spool file: %v", err) + } + }() + _, err = io.Copy(spool, streamIn) + if err != nil { + return nil, fmt.Errorf("failed to copy to temporary spool file: %v", err) + } + rs = spool + } + // Upload with Put with retries - since we have downloaded the file we know the size, and the hashes + sums := getSums() + size := int64(readCounter.BytesRead()) + objInfo := object.NewStaticObjectInfo(dstFileName, modTime, size, false, sums, fsrc).WithMetadata(meta) + err = Retry(ctx, objInfo, ci.LowLevelRetries, func() error { + _, err = rs.Seek(0, io.SeekStart) + if err != nil { + return fmt.Errorf("failed to rewind temporary spool file: %v", err) + } + dst, err = fdst.Put(ctx, rs, objInfo, options...) + return err + }) + } else { + // Upload with PutStream with no retries + objInfo := object.NewStaticObjectInfo(dstFileName, modTime, -1, false, nil, fsrc).WithMetadata(meta) + dst, err = doPutStream(ctx, streamIn, objInfo, options...) + } + if err != nil { + return dst, err + } + + // Check transfer + sums := getSums() + opt := defaultEqualOpt(ctx) + if sums != nil { + // force --checksum on if we have hashes + opt.checkSum = true + } + src := object.NewStaticObjectInfo(dstFileName, modTime, int64(readCounter.BytesRead()), false, sums, fdst).WithMetadata(meta) + if !equal(ctx, src, dst, opt) { + err = fmt.Errorf("corrupted on transfer") + err = fs.CountError(ctx, err) + fs.Errorf(dst, "%v", err) + return dst, err + } + return dst, nil +} + +// PublicLink adds a "readable by anyone with link" permission on the given file or folder. +func PublicLink(ctx context.Context, f fs.Fs, remote string, expire fs.Duration, unlink bool) (string, error) { + doPublicLink := f.Features().PublicLink + if doPublicLink == nil { + return "", fmt.Errorf("%v doesn't support public links", f) + } + return doPublicLink(ctx, remote, expire, unlink) +} + +// Rmdirs removes any empty directories (or directories only +// containing empty directories) under f, including f. +// +// Rmdirs obeys the filters +func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error { + ci := fs.GetConfig(ctx) + fi := filter.GetConfig(ctx) + dirEmpty := make(map[string]bool) + dirEmpty[dir] = !leaveRoot + err := walk.Walk(ctx, f, dir, false, ci.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error { + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(f, "Failed to list %q: %v", dirPath, err) + return nil + } + for _, entry := range entries { + switch x := entry.(type) { + case fs.Directory: + // add a new directory as empty + dir := x.Remote() + _, found := dirEmpty[dir] + if !found { + dirEmpty[dir] = true + } + case fs.Object: + // mark the parents of the file as being non-empty + dir := x.Remote() + for dir != "" { + dir = path.Dir(dir) + if dir == "." || dir == "/" { + dir = "" + } + empty, found := dirEmpty[dir] + // End if we reach a directory which is non-empty + if found && !empty { + break + } + dirEmpty[dir] = false + } + } + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to rmdirs: %w", err) + } + + // Group directories to delete by level + var toDelete [][]string + for dir, empty := range dirEmpty { + if empty { + // If a filter matches the directory then that + // directory is a candidate for deletion + if fi.IncludeRemote(dir + "/") { + level := strings.Count(dir, "/") + 1 + // The root directory "" is at the top level + if dir == "" { + level = 0 + } + if len(toDelete) < level+1 { + toDelete = append(toDelete, make([][]string, level+1-len(toDelete))...) + } + toDelete[level] = append(toDelete[level], dir) + } + } + } + + errCount := errcount.New() + // Delete all directories at the same level in parallel + for level := len(toDelete) - 1; level >= 0; level-- { + dirs := toDelete[level] + if len(dirs) == 0 { + continue + } + fs.Debugf(nil, "removing %d level %d directories", len(dirs), level) + sort.Strings(dirs) + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(ci.Checkers) + for _, dir := range dirs { + // End early if error + if gCtx.Err() != nil { + break + } + dir := dir + g.Go(func() error { + err := TryRmdir(gCtx, f, dir) + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(dir, "Failed to rmdir: %v", err) + errCount.Add(err) + } + return nil // don't return errors, just count them + }) + } + err := g.Wait() + if err != nil { + return err + } + } + return errCount.Err("failed to remove directories") +} + +// GetCompareDest sets up --compare-dest +func GetCompareDest(ctx context.Context) (CompareDest []fs.Fs, err error) { + ci := fs.GetConfig(ctx) + CompareDest, err = cache.GetArr(ctx, ci.CompareDest) + if err != nil { + return nil, fserrors.FatalError(fmt.Errorf("failed to make fs for --compare-dest %q: %w", ci.CompareDest, err)) + } + return CompareDest, nil +} + +// compareDest checks --compare-dest to see if src needs to +// be copied +// +// Returns True if src is in --compare-dest +func compareDest(ctx context.Context, dst, src fs.Object, CompareDest fs.Fs) (NoNeedTransfer bool, err error) { + var remote string + if dst == nil { + remote = src.Remote() + } else { + remote = dst.Remote() + } + CompareDestFile, err := CompareDest.NewObject(ctx, remote) + switch err { + case fs.ErrorObjectNotFound: + return false, nil + case nil: + break + default: + return false, err + } + opt := defaultEqualOpt(ctx) + opt.updateModTime = false + if equal(ctx, src, CompareDestFile, opt) { + fs.Debugf(src, "Destination found in --compare-dest, skipping") + return true, nil + } + return false, nil +} + +// GetCopyDest sets up --copy-dest +func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest []fs.Fs, err error) { + ci := fs.GetConfig(ctx) + CopyDest, err = cache.GetArr(ctx, ci.CopyDest) + if err != nil { + return nil, fserrors.FatalError(fmt.Errorf("failed to make fs for --copy-dest %q: %w", ci.CopyDest, err)) + } + if !SameConfigArr(fdst, CopyDest) { + return nil, fserrors.FatalError(errors.New("parameter to --copy-dest has to be on the same remote as destination")) + } + for _, cf := range CopyDest { + if cf.Features().Copy == nil { + return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server side copy")) + } + } + + return CopyDest, nil +} + +// copyDest checks --copy-dest to see if src needs to +// be copied +// +// Returns True if src was copied from --copy-dest +func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) { + var remote string + if dst == nil { + remote = src.Remote() + } else { + remote = dst.Remote() + } + CopyDestFile, err := CopyDest.NewObject(ctx, remote) + switch err { + case fs.ErrorObjectNotFound: + return false, nil + case nil: + break + default: + return false, err + } + opt := defaultEqualOpt(ctx) + opt.updateModTime = false + if equal(ctx, src, CopyDestFile, opt) { + if dst == nil || !Equal(ctx, src, dst) { + if dst != nil && backupDir != nil { + err = MoveBackupDir(ctx, backupDir, dst) + if err != nil { + return false, fmt.Errorf("moving to --backup-dir failed: %w", err) + } + // If successful zero out the dstObj as it is no longer there + dst = nil + } + _, err := Copy(ctx, fdst, dst, remote, CopyDestFile) + if err != nil { + fs.Errorf(src, "Destination found in --copy-dest, error copying") + return false, nil + } + fs.Debugf(src, "Destination found in --copy-dest, using server-side copy") + return true, nil + } + fs.Debugf(src, "Unchanged skipping") + return true, nil + } + fs.Debugf(src, "Destination not found in --copy-dest") + return false, nil +} + +// CompareOrCopyDest checks --compare-dest and --copy-dest to see if src +// does not need to be copied +// +// Returns True if src does not need to be copied +func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest []fs.Fs, backupDir fs.Fs) (NoNeedTransfer bool, err error) { + ci := fs.GetConfig(ctx) + if len(ci.CompareDest) > 0 { + for _, compareF := range CompareOrCopyDest { + NoNeedTransfer, err := compareDest(ctx, dst, src, compareF) + if NoNeedTransfer || err != nil { + return NoNeedTransfer, err + } + } + } else if len(ci.CopyDest) > 0 { + for _, copyF := range CompareOrCopyDest { + NoNeedTransfer, err := copyDest(ctx, fdst, dst, src, copyF, backupDir) + if NoNeedTransfer || err != nil { + return NoNeedTransfer, err + } + } + } + return false, nil +} + +// NeedTransfer checks to see if src needs to be copied to dst using +// the current config. +// +// Returns a flag which indicates whether the file needs to be +// transferred or not. +func NeedTransfer(ctx context.Context, dst, src fs.Object) bool { + ci := fs.GetConfig(ctx) + logger, _ := GetLogger(ctx) + if dst == nil { + fs.Debugf(src, "Need to transfer - File not found at Destination") + logger(ctx, MissingOnDst, src, nil, nil) + return true + } + // If we should ignore existing files, don't transfer + if ci.IgnoreExisting { + fs.Debugf(src, "Destination exists, skipping") + logger(ctx, Match, src, dst, nil) + return false + } + // If we should upload unconditionally + if ci.IgnoreTimes { + fs.Debugf(src, "Transferring unconditionally as --ignore-times is in use") + logger(ctx, Differ, src, dst, nil) + return true + } + // If UpdateOlder is in effect, skip if dst is newer than src + if ci.UpdateOlder { + srcModTime := src.ModTime(ctx) + dstModTime := dst.ModTime(ctx) + dt := dstModTime.Sub(srcModTime) + // If have a mutually agreed precision then use that + modifyWindow := fs.GetModifyWindow(ctx, dst.Fs(), src.Fs()) + if modifyWindow == fs.ModTimeNotSupported { + // Otherwise use 1 second as a safe default as + // the resolution of the time a file was + // uploaded. + modifyWindow = time.Second + } + switch { + case dt >= modifyWindow: + fs.Debugf(src, "Destination is newer than source, skipping") + logger(ctx, Match, src, dst, nil) + return false + case dt <= -modifyWindow: + // force --checksum on for the check and do update modtimes by default + opt := defaultEqualOpt(ctx) + opt.forceModTimeMatch = true + if equal(ctx, src, dst, opt) { + fs.Debugf(src, "Unchanged skipping") + return false + } + default: + // Do a size only compare unless --checksum is set + opt := defaultEqualOpt(ctx) + opt.sizeOnly = !ci.CheckSum + if equal(ctx, src, dst, opt) { + fs.Debugf(src, "Destination mod time is within %v of source and files identical, skipping", modifyWindow) + return false + } + fs.Debugf(src, "Destination mod time is within %v of source but files differ, transferring", modifyWindow) + } + } else { + // Check to see if changed or not + equalFn, ok := ctx.Value(equalFnKey).(EqualFn) + if ok { + return !equalFn(ctx, src, dst) + } + if Equal(ctx, src, dst) && !SameObject(src, dst) { + fs.Debugf(src, "Unchanged skipping") + return false + } + } + return true +} + +// RcatSize reads data from the Reader until EOF and uploads it to a file on remote. +// Pass in size >=0 if known, <0 if not known +func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modTime time.Time, meta fs.Metadata) (dst fs.Object, err error) { + var obj fs.Object + + if size >= 0 { + var err error + // Size known use Put + tr := accounting.Stats(ctx).NewTransferRemoteSize(dstFileName, size, nil, fdst) + defer func() { + tr.Done(ctx, err) + }() + body := io.NopCloser(in) // we let the server close the body + in := tr.Account(ctx, body) // account the transfer (no buffering) + + if SkipDestructive(ctx, dstFileName, "upload from pipe") { + // prevents "broken pipe" errors + _, err = io.Copy(io.Discard, in) + return nil, err + } + + info := object.NewStaticObjectInfo(dstFileName, modTime, size, true, nil, fdst).WithMetadata(meta) + obj, err = fdst.Put(ctx, in, info) + if err != nil { + fs.Errorf(dstFileName, "Post request put error: %v", err) + + return nil, err + } + } else { + // Size unknown use Rcat + obj, err = Rcat(ctx, fdst, dstFileName, in, modTime, meta) + if err != nil { + fs.Errorf(dstFileName, "Post request rcat error: %v", err) + + return nil, err + } + } + + return obj, nil +} + +// copyURLFunc is called from CopyURLFn +type copyURLFunc func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) + +// copyURLFn copies the data from the url to the function supplied +func copyURLFn(ctx context.Context, dstFileName string, url string, autoFilename, dstFileNameFromHeader bool, fn copyURLFunc) (err error) { + client := fshttp.NewClient(ctx) + resp, err := client.Get(url) + if err != nil { + return err + } + defer fs.CheckClose(resp.Body, &err) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("CopyURL failed: %s", resp.Status) + } + modTime, err := http.ParseTime(resp.Header.Get("Last-Modified")) + if err != nil { + modTime = time.Now() + } + if autoFilename { + if dstFileNameFromHeader { + _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) + headerFilename := path.Base(strings.ReplaceAll(params["filename"], "\\", "/")) + if err != nil || headerFilename == "" { + return fmt.Errorf("CopyURL failed: filename not found in the Content-Disposition header") + } + fs.Debugf(headerFilename, "filename found in Content-Disposition header.") + return fn(ctx, headerFilename, resp.Body, resp.ContentLength, modTime) + } + + dstFileName = path.Base(resp.Request.URL.Path) + if dstFileName == "." || dstFileName == "/" { + return fmt.Errorf("CopyURL failed: file name wasn't found in url") + } + fs.Debugf(dstFileName, "File name found in url") + } + return fn(ctx, dstFileName, resp.Body, resp.ContentLength, modTime) +} + +// CopyURL copies the data from the url to (fdst, dstFileName) +func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, autoFilename, dstFileNameFromHeader bool, noClobber bool) (dst fs.Object, err error) { + err = copyURLFn(ctx, dstFileName, url, autoFilename, dstFileNameFromHeader, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) { + if noClobber { + _, err = fdst.NewObject(ctx, dstFileName) + if err == nil { + return errors.New("CopyURL failed: file already exist") + } + } + dst, err = RcatSize(ctx, fdst, dstFileName, in, size, modTime, nil) + return err + }) + return dst, err +} + +// CopyURLToWriter copies the data from the url to the io.Writer supplied +func CopyURLToWriter(ctx context.Context, url string, out io.Writer) (err error) { + return copyURLFn(ctx, "", url, false, false, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) { + _, err = io.Copy(out, in) + return err + }) +} + +// BackupDir returns the correctly configured --backup-dir +func BackupDir(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, srcFileName string) (backupDir fs.Fs, err error) { + ci := fs.GetConfig(ctx) + if ci.BackupDir != "" { + backupDir, err = cache.Get(ctx, ci.BackupDir) + if err != nil { + return nil, fserrors.FatalError(fmt.Errorf("failed to make fs for --backup-dir %q: %w", ci.BackupDir, err)) + } + if !SameConfig(fdst, backupDir) { + return nil, fserrors.FatalError(errors.New("parameter to --backup-dir has to be on the same remote as destination")) + } + if srcFileName == "" { + if OverlappingFilterCheck(ctx, backupDir, fdst) { + return nil, fserrors.FatalError(errors.New("destination and parameter to --backup-dir mustn't overlap")) + } + if OverlappingFilterCheck(ctx, backupDir, fsrc) { + return nil, fserrors.FatalError(errors.New("source and parameter to --backup-dir mustn't overlap")) + } + } else if ci.Suffix == "" { + if SameDir(fdst, backupDir) { + return nil, fserrors.FatalError(errors.New("destination and parameter to --backup-dir mustn't be the same")) + } + if SameDir(fsrc, backupDir) { + return nil, fserrors.FatalError(errors.New("source and parameter to --backup-dir mustn't be the same")) + } + } + } else if ci.Suffix != "" { + // --backup-dir is not set but --suffix is - use the destination as the backupDir + backupDir = fdst + } else { + return nil, fserrors.FatalError(errors.New("internal error: BackupDir called when --backup-dir and --suffix both empty")) + } + if !CanServerSideMove(backupDir) { + return nil, fserrors.FatalError(errors.New("can't use --backup-dir on a remote which doesn't support server-side move or copy")) + } + return backupDir, nil +} + +// MoveBackupDir moves a file to the backup dir +func MoveBackupDir(ctx context.Context, backupDir fs.Fs, dst fs.Object) (err error) { + remoteWithSuffix := SuffixName(ctx, dst.Remote()) + overwritten, _ := backupDir.NewObject(ctx, remoteWithSuffix) + _, err = Move(ctx, backupDir, overwritten, remoteWithSuffix, dst) + return err +} + +// needsMoveCaseInsensitive returns true if moveCaseInsensitive is needed +func needsMoveCaseInsensitive(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) bool { + dstFilePath := path.Join(fdst.Root(), dstFileName) + srcFilePath := path.Join(fsrc.Root(), srcFileName) + if !cp && fdst.Name() == fsrc.Name() && dstFileName != srcFileName && norm.NFC.String(dstFilePath) == norm.NFC.String(srcFilePath) { + return true + } + return !cp && fdst.Name() == fsrc.Name() && fdst.Features().CaseInsensitive && dstFileName != srcFileName && strings.EqualFold(dstFilePath, srcFilePath) +} + +// MoveCaseInsensitive handles changing case of a file on a case insensitive remote. +// This will move the file to a temporary name then +// move it back to the intended destination. This is required +// to avoid issues with certain remotes and avoid file deletion. +// returns nil, nil if !needsMoveCaseInsensitive. +// this does not account a transfer -- the caller should do that if desired. +func MoveCaseInsensitive(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool, srcObj fs.Object) (newDst fs.Object, err error) { + logger, _ := GetLogger(ctx) + + // Choose operations + Op := MoveTransfer + if cp { + Op = Copy + } + + if SkipDestructive(ctx, srcFileName, "rename to "+dstFileName) { + // avoid fatalpanic on --dry-run (trying to access non-existent tmpObj) + return nil, nil + } + // Create random name to temporarily move file to + tmpObjName := dstFileName + "-rclone-move-" + random.String(8) + tmpObjFail, err := fdst.NewObject(ctx, tmpObjName) + if err != fs.ErrorObjectNotFound { + if err == nil { + logger(ctx, TransferError, nil, tmpObjFail, err) + return nil, errors.New("found an already existing file with a randomly generated name. Try the operation again") + } + logger(ctx, TransferError, nil, tmpObjFail, err) + return nil, fmt.Errorf("error while attempting to move file to a temporary location: %w", err) + } + fs.Debugf(srcObj, "moving to %v", tmpObjName) + tmpObj, err := Op(ctx, fdst, nil, tmpObjName, srcObj) + if err != nil { + logger(ctx, TransferError, srcObj, tmpObj, err) + return nil, fmt.Errorf("error while moving file to temporary location: %w", err) + } + fs.Debugf(srcObj, "moving to %v", dstFileName) + newDst, err = Op(ctx, fdst, nil, dstFileName, tmpObj) + logger(ctx, MissingOnDst, tmpObj, nil, err) + return newDst, err +} + +// moveOrCopyFile moves or copies a single file possibly to a new name +func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool, allowOverlap bool) (err error) { + ci := fs.GetConfig(ctx) + logger, usingLogger := GetLogger(ctx) + dstFilePath := path.Join(fdst.Root(), dstFileName) + srcFilePath := path.Join(fsrc.Root(), srcFileName) + if fdst.Name() == fsrc.Name() && dstFilePath == srcFilePath && !allowOverlap { + fs.Debugf(fdst, "don't need to copy/move %s, it is already at target location", dstFileName) + if usingLogger { + srcObj, _ := fsrc.NewObject(ctx, srcFileName) + dstObj, _ := fsrc.NewObject(ctx, dstFileName) + logger(ctx, Match, srcObj, dstObj, nil) + } + return nil + } + + // Choose operations + Op := MoveTransfer + if cp { + Op = Copy + } + + // Find src object + srcObj, err := fsrc.NewObject(ctx, srcFileName) + if err != nil { + logger(ctx, TransferError, srcObj, nil, err) + return err + } + + // Find dst object if it exists + var dstObj fs.Object + if !ci.NoCheckDest { + dstObj, err = fdst.NewObject(ctx, dstFileName) + if errors.Is(err, fs.ErrorObjectNotFound) { + dstObj = nil + } else if err != nil { + logger(ctx, TransferError, nil, dstObj, err) + return err + } + } + + // Special case for changing case of a file on a case insensitive remote + // This will move the file to a temporary name then + // move it back to the intended destination. This is required + // to avoid issues with certain remotes and avoid file deletion. + if needsMoveCaseInsensitive(fdst, fsrc, dstFileName, srcFileName, cp) { + tr := accounting.Stats(ctx).NewTransfer(srcObj, fdst) + defer func() { + tr.Done(ctx, err) + }() + _, err = MoveCaseInsensitive(ctx, fdst, fsrc, dstFileName, srcFileName, cp, srcObj) + return err + } + + var backupDir fs.Fs + var copyDestDir []fs.Fs + if ci.BackupDir != "" || ci.Suffix != "" { + backupDir, err = BackupDir(ctx, fdst, fsrc, srcFileName) + if err != nil { + return fmt.Errorf("creating Fs for --backup-dir failed: %w", err) + } + } + if len(ci.CompareDest) > 0 { + copyDestDir, err = GetCompareDest(ctx) + if err != nil { + return err + } + } else if len(ci.CopyDest) > 0 { + copyDestDir, err = GetCopyDest(ctx, fdst) + if err != nil { + return err + } + } + needTransfer := NeedTransfer(ctx, dstObj, srcObj) + if needTransfer { + NoNeedTransfer, err := CompareOrCopyDest(ctx, fdst, dstObj, srcObj, copyDestDir, backupDir) + if err != nil { + return err + } + if NoNeedTransfer { + needTransfer = false + } + } + if needTransfer { + // If destination already exists, then we must move it into --backup-dir if required + if dstObj != nil && backupDir != nil { + err = MoveBackupDir(ctx, backupDir, dstObj) + if err != nil { + logger(ctx, TransferError, dstObj, nil, err) + return fmt.Errorf("moving to --backup-dir failed: %w", err) + } + // If successful zero out the dstObj as it is no longer there + logger(ctx, MissingOnDst, dstObj, nil, nil) + dstObj = nil + } + + _, err = Op(ctx, fdst, dstObj, dstFileName, srcObj) + } else if !cp { + if ci.IgnoreExisting { + fs.Debugf(srcObj, "Not removing source file as destination file exists and --ignore-existing is set") + logger(ctx, Match, srcObj, dstObj, nil) + } else if !SameObject(srcObj, dstObj) { + err = DeleteFile(ctx, srcObj) + logger(ctx, Differ, srcObj, dstObj, nil) + } + } + return err +} + +// MoveFile moves a single file possibly to a new name +// +// This is treated as a transfer. +func MoveFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) { + return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, false, false) +} + +// TransformFile transforms a file in place using --name-transform +// +// This is treated as a transfer. +func TransformFile(ctx context.Context, fdst fs.Fs, srcFileName string) (err error) { + return moveOrCopyFile(ctx, fdst, fdst, srcFileName, srcFileName, false, true) +} + +// SetTier changes tier of object in remote +func SetTier(ctx context.Context, fsrc fs.Fs, tier string) error { + return ListFn(ctx, fsrc, func(o fs.Object) { + objImpl, ok := o.(fs.SetTierer) + if !ok { + fs.Errorf(fsrc, "Remote object does not implement SetTier") + return + } + err := objImpl.SetTier(tier) + if err != nil { + fs.Errorf(fsrc, "Failed to do SetTier, %v", err) + } + }) +} + +// SetTierFile changes tier of a single file in remote +func SetTierFile(ctx context.Context, o fs.Object, tier string) error { + do, ok := o.(fs.SetTierer) + if !ok { + return errors.New("remote object does not implement SetTier") + } + err := do.SetTier(tier) + if err != nil { + fs.Errorf(o, "Failed to do SetTier, %v", err) + return err + } + return nil +} + +// TouchDir touches every file in directory with time t +func TouchDir(ctx context.Context, f fs.Fs, remote string, t time.Time, recursive bool) error { + ci := fs.GetConfig(ctx) + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(ci.Transfers) + err := walk.ListR(ctx, f, remote, false, ConfigMaxDepth(ctx, recursive), walk.ListObjects, func(entries fs.DirEntries) error { + entries.ForObject(func(o fs.Object) { + if !SkipDestructive(ctx, o, "touch") { + g.Go(func() error { + fs.Debugf(f, "Touching %q", o.Remote()) + err := o.SetModTime(gCtx, t) + if err != nil { + err = fmt.Errorf("failed to touch: %w", err) + err = fs.CountError(gCtx, err) + fs.Errorf(o, "%v", err) + } + return nil + }) + } + }) + return nil + }) + _ = g.Wait() + return err +} + +// ListFormat defines files information print format +type ListFormat struct { + separator string + dirSlash bool + absolute bool + output []func(entry *ListJSONItem) string + csv *csv.Writer + buf bytes.Buffer +} + +// SetSeparator changes separator in struct +func (l *ListFormat) SetSeparator(separator string) { + l.separator = separator +} + +// SetDirSlash defines if slash should be printed +func (l *ListFormat) SetDirSlash(dirSlash bool) { + l.dirSlash = dirSlash +} + +// SetAbsolute prints a leading slash in front of path names +func (l *ListFormat) SetAbsolute(absolute bool) { + l.absolute = absolute +} + +// SetCSV defines if the output should be csv +// +// Note that you should call SetSeparator before this if you want a +// custom separator +func (l *ListFormat) SetCSV(useCSV bool) { + if useCSV { + l.csv = csv.NewWriter(&l.buf) + if l.separator != "" { + l.csv.Comma = []rune(l.separator)[0] + } + } else { + l.csv = nil + } +} + +// SetOutput sets functions used to create files information +func (l *ListFormat) SetOutput(output []func(entry *ListJSONItem) string) { + l.output = output +} + +// AddModTime adds file's Mod Time to output +func (l *ListFormat) AddModTime(timeFormat string) { + switch timeFormat { + case "": + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.ModTime.When.Local().Format("2006-01-02 15:04:05") + }) + case "unix": + l.AppendOutput(func(entry *ListJSONItem) string { + return fmt.Sprint(entry.ModTime.When.Unix()) + }) + case "unixnano": + l.AppendOutput(func(entry *ListJSONItem) string { + return fmt.Sprint(entry.ModTime.When.UnixNano()) + }) + default: + timeFormat = transform.TimeFormat(timeFormat) + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.ModTime.When.Local().Format(timeFormat) + }) + } +} + +// AddSize adds file's size to output +func (l *ListFormat) AddSize() { + l.AppendOutput(func(entry *ListJSONItem) string { + return strconv.FormatInt(entry.Size, 10) + }) +} + +// normalisePath makes sure the path has the correct slashes for the current mode +func (l *ListFormat) normalisePath(entry *ListJSONItem, remote string) string { + if l.absolute && !strings.HasPrefix(remote, "/") { + remote = "/" + remote + } + if entry.IsDir && l.dirSlash { + remote += "/" + } + return remote +} + +// AddPath adds path to file to output +func (l *ListFormat) AddPath() { + l.AppendOutput(func(entry *ListJSONItem) string { + return l.normalisePath(entry, entry.Path) + }) +} + +// AddEncrypted adds the encrypted path to file to output +func (l *ListFormat) AddEncrypted() { + l.AppendOutput(func(entry *ListJSONItem) string { + return l.normalisePath(entry, entry.Encrypted) + }) +} + +// AddHash adds the hash of the type given to the output +func (l *ListFormat) AddHash(ht hash.Type) { + hashName := ht.String() + l.AppendOutput(func(entry *ListJSONItem) string { + if entry.IsDir { + return "" + } + return entry.Hashes[hashName] + }) +} + +// AddID adds file's ID to the output if known +func (l *ListFormat) AddID() { + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.ID + }) +} + +// AddOrigID adds file's Original ID to the output if known +func (l *ListFormat) AddOrigID() { + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.OrigID + }) +} + +// AddTier adds file's Tier to the output if known +func (l *ListFormat) AddTier() { + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.Tier + }) +} + +// AddMimeType adds file's MimeType to the output if known +func (l *ListFormat) AddMimeType() { + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.MimeType + }) +} + +// AddMetadata adds file's Metadata to the output if known +func (l *ListFormat) AddMetadata() { + l.AppendOutput(func(entry *ListJSONItem) string { + metadata := entry.Metadata + if metadata == nil { + metadata = make(fs.Metadata) + } + out, err := json.Marshal(metadata) + if err != nil { + return fmt.Sprintf("Failed to read metadata: %v", err.Error()) + } + return string(out) + }) +} + +// AppendOutput adds string generated by specific function to printed output +func (l *ListFormat) AppendOutput(functionToAppend func(item *ListJSONItem) string) { + l.output = append(l.output, functionToAppend) +} + +// Format prints information about the DirEntry in the format defined +func (l *ListFormat) Format(entry *ListJSONItem) (result string) { + var out []string + for _, fun := range l.output { + out = append(out, fun(entry)) + } + if l.csv != nil { + l.buf.Reset() + _ = l.csv.Write(out) // can't fail writing to bytes.Buffer + l.csv.Flush() + result = strings.TrimRight(l.buf.String(), "\n") + } else { + result = strings.Join(out, l.separator) + } + return result +} + +// FormatForLSFPrecision Returns a time format for the given precision +func FormatForLSFPrecision(precision time.Duration) string { + switch { + case precision <= time.Nanosecond: + return "2006-01-02 15:04:05.000000000" + case precision <= 10*time.Nanosecond: + return "2006-01-02 15:04:05.00000000" + case precision <= 100*time.Nanosecond: + return "2006-01-02 15:04:05.0000000" + case precision <= time.Microsecond: + return "2006-01-02 15:04:05.000000" + case precision <= 10*time.Microsecond: + return "2006-01-02 15:04:05.00000" + case precision <= 100*time.Microsecond: + return "2006-01-02 15:04:05.0000" + case precision <= time.Millisecond: + return "2006-01-02 15:04:05.000" + case precision <= 10*time.Millisecond: + return "2006-01-02 15:04:05.00" + case precision <= 100*time.Millisecond: + return "2006-01-02 15:04:05.0" + } + return "2006-01-02 15:04:05" +} + +// DirMove renames srcRemote to dstRemote +// +// It does this by loading the directory tree into memory (using ListR +// if available) and doing renames in parallel. +func DirMove(ctx context.Context, f fs.Fs, srcRemote, dstRemote string) (err error) { + ci := fs.GetConfig(ctx) + + if SkipDestructive(ctx, srcRemote, "dirMove") { + accounting.Stats(ctx).Renames(1) + return nil + } + + // Use DirMove if possible + if doDirMove := f.Features().DirMove; doDirMove != nil { + err = doDirMove(ctx, f, srcRemote, dstRemote) + if err == nil { + accounting.Stats(ctx).Renames(1) + } + if err != fs.ErrorCantDirMove && err != fs.ErrorDirExists { + return err + } + fs.Infof(f, "Can't DirMove - falling back to file moves: %v", err) + } + + // Load the directory tree into memory + tree, err := walk.NewDirTree(ctx, f, srcRemote, true, -1) + if err != nil { + return fmt.Errorf("RenameDir tree walk: %w", err) + } + + // Get the directories in sorted order + dirs := tree.Dirs() + + // Make the destination directories - must be done in order not in parallel + for _, dir := range dirs { + dstPath := dstRemote + dir[len(srcRemote):] + err := f.Mkdir(ctx, dstPath) + if err != nil { + return fmt.Errorf("RenameDir mkdir: %w", err) + } + } + + // Rename the files in parallel + type rename struct { + o fs.Object + newPath string + } + renames := make(chan rename, ci.Checkers) + g, gCtx := errgroup.WithContext(context.Background()) + for range ci.Checkers { + g.Go(func() error { + for job := range renames { + dstOverwritten, _ := f.NewObject(gCtx, job.newPath) + _, err := Move(gCtx, f, dstOverwritten, job.newPath, job.o) + if err != nil { + return err + } + select { + case <-gCtx.Done(): + return gCtx.Err() + default: + } + + } + return nil + }) + } + for dir, entries := range tree { + dstPath := dstRemote + dir[len(srcRemote):] + for _, entry := range entries { + if o, ok := entry.(fs.Object); ok { + renames <- rename{o, path.Join(dstPath, path.Base(o.Remote()))} + } + } + } + close(renames) + err = g.Wait() + if err != nil { + return fmt.Errorf("RenameDir renames: %w", err) + } + + // Remove the source directories in reverse order + for i := len(dirs) - 1; i >= 0; i-- { + err := f.Rmdir(ctx, dirs[i]) + if err != nil { + return fmt.Errorf("RenameDir rmdir: %w", err) + } + } + + return nil +} + +// DirMoveCaseInsensitive does DirMove in two steps (to temp name, then real name) +// which is necessary for some case-insensitive backends +func DirMoveCaseInsensitive(ctx context.Context, f fs.Fs, srcRemote, dstRemote string) (err error) { + tmpDstRemote := dstRemote + "-rclone-move-" + random.String(8) + err = DirMove(ctx, f, srcRemote, tmpDstRemote) + if err != nil { + return err + } + return DirMove(ctx, f, tmpDstRemote, dstRemote) +} + +// FsInfo provides information about a remote +type FsInfo struct { + // Name of the remote (as passed into NewFs) + Name string + + // Root of the remote (as passed into NewFs) + Root string + + // String returns a description of the FS + String string + + // Precision of the ModTimes in this Fs in Nanoseconds + Precision time.Duration + + // Returns the supported hash types of the filesystem + Hashes []string + + // Features returns the optional features of this Fs + Features map[string]bool + + // MetadataInfo returns info about the metadata for this backend + MetadataInfo *fs.MetadataInfo +} + +// GetFsInfo gets the information (FsInfo) about a given Fs +func GetFsInfo(f fs.Fs) *FsInfo { + features := f.Features() + info := &FsInfo{ + Name: f.Name(), + Root: f.Root(), + String: f.String(), + Precision: f.Precision(), + Hashes: make([]string, 0, 4), + Features: features.Enabled(), + MetadataInfo: nil, + } + for _, hashType := range f.Hashes().Array() { + info.Hashes = append(info.Hashes, hashType.String()) + } + fsInfo, _, _, _, err := fs.ParseRemote(fs.ConfigString(f)) + if err == nil && fsInfo != nil && fsInfo.MetadataInfo != nil { + info.MetadataInfo = fsInfo.MetadataInfo + } + return info +} + +var ( + interactiveMu sync.Mutex // protects the following variables + skipped = map[string]bool{} +) + +// skipDestructiveChoose asks the user which action to take +// +// Call with interactiveMu held +func skipDestructiveChoose(ctx context.Context, subject any, action string) (skip bool) { + // Lock the StdoutMutex - must not call fs.Log anything + // otherwise it will deadlock with --interactive --progress + StdoutMutex.Lock() + + fmt.Printf("\nrclone: %s \"%v\"?\n", action, subject) + i := config.CommandDefault([]string{ + "yYes, this is OK", + "nNo, skip this", + fmt.Sprintf("sSkip all %s operations with no more questions", action), + fmt.Sprintf("!Do all %s operations with no more questions", action), + "qExit rclone now.", + }, 0) + + StdoutMutex.Unlock() + + switch i { + case 'y': + skip = false + case 'n': + skip = true + case 's': + skip = true + skipped[action] = true + fs.Logf(nil, "Skipping all %s operations from now on without asking", action) + case '!': + skip = false + skipped[action] = false + fs.Logf(nil, "Doing all %s operations from now on without asking", action) + case 'q': + fs.Logf(nil, "Quitting rclone now") + atexit.Run() + os.Exit(0) + default: + skip = true + fs.Errorf(nil, "Bad choice %c", i) + } + return skip +} + +// SkipDestructive should be called whenever rclone is about to do an destructive operation. +// +// It will check the --dry-run flag and it will ask the user if the --interactive flag is set. +// +// subject should be the object or directory in use +// +// action should be a descriptive word or short phrase +// +// Together they should make sense in this sentence: "Rclone is about +// to action subject". +func SkipDestructive(ctx context.Context, subject any, action string) (skip bool) { + var flag string + ci := fs.GetConfig(ctx) + switch { + case ci.DryRun: + flag = "--dry-run" + skip = true + case ci.Interactive: + flag = "--interactive" + interactiveMu.Lock() + defer interactiveMu.Unlock() + var found bool + skip, found = skipped[action] + if !found { + skip = skipDestructiveChoose(ctx, subject, action) + } + default: + return false + } + if skip { + size := int64(-1) + if do, ok := subject.(interface{ Size() int64 }); ok { + size = do.Size() + } + if size >= 0 { + fs.Logf(subject, "Skipped %s as %s is set (size %v)", fs.LogValue("skipped", action), flag, fs.LogValue("size", fs.SizeSuffix(size))) + } else { + fs.Logf(subject, "Skipped %s as %s is set", fs.LogValue("skipped", action), flag) + } + } + return skip +} + +// Return the best way of describing the directory for the logs +func dirName(f fs.Fs, dst fs.Directory, dir string) any { + if dst != nil { + if dst.Remote() != "" { + return dst + } + // Root is described as the Fs + return f + } + if dir != "" { + return dir + } + // Root is described as the Fs + return f +} + +// CopyDirMetadata copies the src directory to dst or f if nil. If dst is nil then it uses +// dir as the name of the new directory. +// +// It returns the destination directory if possible. Note that this may +// be nil. +func CopyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory, err error) { + ci := fs.GetConfig(ctx) + logName := dirName(f, dst, dir) + if SkipDestructive(ctx, logName, "update directory metadata") { + return nil, nil + } + + // Options for the directory metadata + options := []fs.OpenOption{} + if ci.MetadataSet != nil { + options = append(options, fs.MetadataOption(ci.MetadataSet)) + } + + // Read metadata from src and add options and use metadata mapper + metadata, err := fs.GetMetadataOptions(ctx, f, src, options) + if err != nil { + return nil, err + } + + // Fall back to ModTime if metadata not available + if metadata == nil { + metadata = fs.Metadata{} + } + if metadata["mtime"] == "" { + metadata["mtime"] = src.ModTime(ctx).Format(time.RFC3339Nano) + } + + // Now set the metadata + if dst == nil { + do := f.Features().MkdirMetadata + if do == nil { + return nil, fmt.Errorf("internal error: expecting %v to have MkdirMetadata method: %w", f, fs.ErrorNotImplemented) + } + newDst, err = do(ctx, dir, metadata) + } else { + do, ok := dst.(fs.SetMetadataer) + if !ok { + return nil, fmt.Errorf("internal error: expecting directory %s (%T) from %v to have SetMetadata method: %w", logName, dst, f, fs.ErrorNotImplemented) + } + err = do.SetMetadata(ctx, metadata) + newDst = dst + } + if err != nil { + return nil, err + } + fs.Infof(logName, "Updated directory metadata") + return newDst, nil +} + +// SetDirModTime sets the modtime on dst or dir +// +// If dst is nil then it uses dir as the name of the directory. +// +// It returns the destination directory if possible. Note that this +// may be nil. +// +// It does not create the directory. +func SetDirModTime(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, modTime time.Time) (newDst fs.Directory, err error) { + logName := dirName(f, dst, dir) + ci := fs.GetConfig(ctx) + if ci.NoUpdateDirModTime { + fs.Debugf(logName, "Skipping set directory modification time as --no-update-dir-modtime is set") + return nil, nil + } + if SkipDestructive(ctx, logName, "set directory modification time") { + return nil, nil + } + if dst != nil { + dir = dst.Remote() + } + + // Try to set the ModTime with the Directory.SetModTime method first as this is the most efficient + if dst != nil { + if do, ok := dst.(fs.SetModTimer); ok { + err := do.SetModTime(ctx, modTime) + if errors.Is(err, fs.ErrorNotImplemented) { + // Fall through and run the code below if not implemented + // This can happen for fs.DirWrapper instances + } else if err != nil { + return dst, err + } else { + fs.Infof(logName, "Set directory modification time (using SetModTime)") + return dst, nil + } + } + } + + // Next try to set the ModTime with the Fs.DirSetModTime method as this works for non-metadata backends + if do := f.Features().DirSetModTime; do != nil { + err := do(ctx, dir, modTime) + if err != nil { + return dst, err + } + fs.Infof(logName, "Set directory modification time (using DirSetModTime)") + return dst, nil + } + + // Something should have worked so return an error + return nil, fmt.Errorf("no method to set directory modtime found for %v (%T): %w", f, dst, fs.ErrorNotImplemented) +} diff --git a/fs/operations/operations_internal_test.go b/fs/operations/operations_internal_test.go new file mode 100644 index 0000000..658249e --- /dev/null +++ b/fs/operations/operations_internal_test.go @@ -0,0 +1,44 @@ +// Internal tests for operations + +package operations + +import ( + "context" + "fmt" + "testing" + "time" + + _ "github.com/mewsen/rclone-studip-backend-oot/backend/studip" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/object" + "github.com/stretchr/testify/assert" +) + +func TestSizeDiffers(t *testing.T) { + ctx := context.Background() + ci := fs.GetConfig(ctx) + when := time.Now() + for _, test := range []struct { + ignoreSize bool + srcSize int64 + dstSize int64 + want bool + }{ + {false, 0, 0, false}, + {false, 1, 2, true}, + {false, 1, -1, false}, + {false, -1, 1, false}, + {true, 0, 0, false}, + {true, 1, 2, false}, + {true, 1, -1, false}, + {true, -1, 1, false}, + } { + src := object.NewStaticObjectInfo("a", when, test.srcSize, true, nil, nil) + dst := object.NewStaticObjectInfo("a", when, test.dstSize, true, nil, nil) + oldIgnoreSize := ci.IgnoreSize + ci.IgnoreSize = test.ignoreSize + got := sizeDiffers(ctx, src, dst) + ci.IgnoreSize = oldIgnoreSize + assert.Equal(t, test.want, got, fmt.Sprintf("ignoreSize=%v, srcSize=%v, dstSize=%v", test.ignoreSize, test.srcSize, test.dstSize)) + } +} diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go new file mode 100644 index 0000000..3fb5992 --- /dev/null +++ b/fs/operations/operations_test.go @@ -0,0 +1,1968 @@ +// Integration tests - test rclone by doing real transactions to a +// storage provider to and from the local disk. +// +// By default it will use a local fs, however you can provide a +// -remote option to use a different remote. The test_all.go script +// is a wrapper to call this for all the test remotes. +// +// FIXME not safe for concurrent running of tests until fs.Config is +// no longer a global +// +// NB When writing tests +// +// Make sure every series of writes to the remote has a +// fstest.CheckItems() before use. This make sure the directory +// listing is now consistent and stops cascading errors. +// +// Call accounting.GlobalStats().ResetCounters() before every fs.Sync() as it +// uses the error count internally. + +package operations_test + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "regexp" + "strings" + "testing" + "time" + + _ "github.com/mewsen/rclone-studip-backend-oot/backend/studip" + _ "github.com/rclone/rclone/backend/all" // import all backends + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/fstest/fstests" + "github.com/rclone/rclone/lib/pacer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/text/cases" + "golang.org/x/text/language" +) + +// Some times used in the tests +var ( + t1 = fstest.Time("2001-02-03T04:05:06.499999999Z") + t2 = fstest.Time("2011-12-25T12:59:59.123456789Z") + t3 = fstest.Time("2011-12-30T12:59:59.000000000Z") +) + +// TestMain drives the tests +func TestMain(m *testing.M) { + fstest.TestMain(m) +} + +func TestMkdir(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + + err := operations.Mkdir(ctx, r.Fremote, "") + require.NoError(t, err) + fstest.CheckListing(t, r.Fremote, []fstest.Item{}) + + err = operations.Mkdir(ctx, r.Fremote, "") + require.NoError(t, err) +} + +func TestLsd(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1) + + r.CheckRemoteItems(t, file1) + + var buf bytes.Buffer + err := operations.ListDir(ctx, r.Fremote, &buf) + require.NoError(t, err) + res := buf.String() + assert.Contains(t, res, "sub dir\n") +} + +func TestLs(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) + file2 := r.WriteBoth(ctx, "empty space", "-", t2) + + r.CheckRemoteItems(t, file1, file2) + + var buf bytes.Buffer + err := operations.List(ctx, r.Fremote, &buf) + require.NoError(t, err) + res := buf.String() + assert.Contains(t, res, " 1 empty space\n") + assert.Contains(t, res, " 60 potato2\n") +} + +func TestLsWithFilesFrom(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) + file2 := r.WriteBoth(ctx, "empty space", "-", t2) + + r.CheckRemoteItems(t, file1, file2) + + // Set the --files-from equivalent + f, err := filter.NewFilter(nil) + require.NoError(t, err) + require.NoError(t, f.AddFile("potato2")) + require.NoError(t, f.AddFile("notfound")) + + // Change the active filter + ctx = filter.ReplaceConfig(ctx, f) + + var buf bytes.Buffer + err = operations.List(ctx, r.Fremote, &buf) + require.NoError(t, err) + assert.Equal(t, " 60 potato2\n", buf.String()) + + // Now try with --no-traverse + ci.NoTraverse = true + + buf.Reset() + err = operations.List(ctx, r.Fremote, &buf) + require.NoError(t, err) + assert.Equal(t, " 60 potato2\n", buf.String()) +} + +func TestLsLong(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) + file2 := r.WriteBoth(ctx, "empty space", "-", t2) + + r.CheckRemoteItems(t, file1, file2) + + var buf bytes.Buffer + err := operations.ListLong(ctx, r.Fremote, &buf) + require.NoError(t, err) + res := buf.String() + lines := strings.Split(strings.Trim(res, "\n"), "\n") + assert.Equal(t, 2, len(lines)) + + timeFormat := "2006-01-02 15:04:05.000000000" + precision := r.Fremote.Precision() + location := time.Now().Location() + checkTime := func(m, filename string, expected time.Time) { + modTime, err := time.ParseInLocation(timeFormat, m, location) // parse as localtime + if err != nil { + t.Errorf("Error parsing %q: %v", m, err) + } else { + fstest.AssertTimeEqualWithPrecision(t, filename, expected, modTime, precision) + } + } + + m1 := regexp.MustCompile(`(?m)^ 1 (\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{9}) empty space$`) + if ms := m1.FindStringSubmatch(res); ms == nil { + t.Errorf("empty space missing: %q", res) + } else { + checkTime(ms[1], "empty space", t2.Local()) + } + + m2 := regexp.MustCompile(`(?m)^ 60 (\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{9}) potato2$`) + if ms := m2.FindStringSubmatch(res); ms == nil { + t.Errorf("potato2 missing: %q", res) + } else { + checkTime(ms[1], "potato2", t1.Local()) + } +} + +func TestHashSums(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) + file2 := r.WriteBoth(ctx, "empty space", "-", t2) + + r.CheckRemoteItems(t, file1, file2) + + hashes := r.Fremote.Hashes() + + var quickXorHash hash.Type + err := quickXorHash.Set("QuickXorHash") + require.NoError(t, err) + + for _, test := range []struct { + name string + download bool + base64 bool + ht hash.Type + want []string + }{ + { + ht: hash.MD5, + want: []string{ + "336d5ebc5436534e61d16e63ddfca327 empty space\n", + "d6548b156ea68a4e003e786df99eee76 potato2\n", + }, + }, + { + ht: hash.MD5, + download: true, + want: []string{ + "336d5ebc5436534e61d16e63ddfca327 empty space\n", + "d6548b156ea68a4e003e786df99eee76 potato2\n", + }, + }, + { + ht: hash.SHA1, + want: []string{ + "3bc15c8aae3e4124dd409035f32ea2fd6835efc9 empty space\n", + "9dc7f7d3279715991a22853f5981df582b7f9f6d potato2\n", + }, + }, + { + ht: hash.SHA1, + download: true, + want: []string{ + "3bc15c8aae3e4124dd409035f32ea2fd6835efc9 empty space\n", + "9dc7f7d3279715991a22853f5981df582b7f9f6d potato2\n", + }, + }, + { + ht: quickXorHash, + want: []string{ + "2d00000000000000000000000100000000000000 empty space\n", + "4001dad296b6b4a52d6d694b67dad296b6b4a52d potato2\n", + }, + }, + { + ht: quickXorHash, + download: true, + want: []string{ + "2d00000000000000000000000100000000000000 empty space\n", + "4001dad296b6b4a52d6d694b67dad296b6b4a52d potato2\n", + }, + }, + { + ht: quickXorHash, + base64: true, + want: []string{ + "LQAAAAAAAAAAAAAAAQAAAAAAAAA= empty space\n", + "QAHa0pa2tKUtbWlLZ9rSlra0pS0= potato2\n", + }, + }, + { + ht: quickXorHash, + base64: true, + download: true, + want: []string{ + "LQAAAAAAAAAAAAAAAQAAAAAAAAA= empty space\n", + "QAHa0pa2tKUtbWlLZ9rSlra0pS0= potato2\n", + }, + }, + } { + if !hashes.Contains(test.ht) { + continue + } + name := cases.Title(language.Und, cases.NoLower).String(test.ht.String()) + if test.download { + name += "Download" + } + if test.base64 { + name += "Base64" + } + t.Run(name, func(t *testing.T) { + var buf bytes.Buffer + err := operations.HashLister(ctx, test.ht, test.base64, test.download, r.Fremote, &buf) + require.NoError(t, err) + res := buf.String() + for _, line := range test.want { + assert.Contains(t, res, line) + } + }) + } +} + +func TestHashSumsWithErrors(t *testing.T) { + ctx := context.Background() + memFs, err := fs.NewFs(ctx, ":memory:") + require.NoError(t, err) + + // Make a test file + content := "-" + item1 := fstest.NewItem("file1", content, t1) + _ = fstests.PutTestContents(ctx, t, memFs, &item1, content, true) + + // MemoryFS supports MD5 + buf := &bytes.Buffer{} + err = operations.HashLister(ctx, hash.MD5, false, false, memFs, buf) + require.NoError(t, err) + assert.Contains(t, buf.String(), "336d5ebc5436534e61d16e63ddfca327 file1\n") + + // MemoryFS can't do SHA1, but UNSUPPORTED must not appear in the output + buf.Reset() + err = operations.HashLister(ctx, hash.SHA1, false, false, memFs, buf) + require.NoError(t, err) + assert.NotContains(t, buf.String(), " UNSUPPORTED ") + + // ERROR must not appear in the output either + assert.NotContains(t, buf.String(), " ERROR ") + // TODO mock an unreadable file +} + +func TestHashStream(t *testing.T) { + reader := strings.NewReader("") + in := io.NopCloser(reader) + out := &bytes.Buffer{} + for _, test := range []struct { + input string + ht hash.Type + wantHex string + wantBase64 string + }{ + { + input: "", + ht: hash.MD5, + wantHex: "d41d8cd98f00b204e9800998ecf8427e -\n", + wantBase64: "1B2M2Y8AsgTpgAmY7PhCfg== -\n", + }, + { + input: "", + ht: hash.SHA1, + wantHex: "da39a3ee5e6b4b0d3255bfef95601890afd80709 -\n", + wantBase64: "2jmj7l5rSw0yVb_vlWAYkK_YBwk= -\n", + }, + { + input: "Hello world!", + ht: hash.MD5, + wantHex: "86fb269d190d2c85f6e0468ceca42a20 -\n", + wantBase64: "hvsmnRkNLIX24EaM7KQqIA== -\n", + }, + { + input: "Hello world!", + ht: hash.SHA1, + wantHex: "d3486ae9136e7856bc42212385ea797094475802 -\n", + wantBase64: "00hq6RNueFa8QiEjhep5cJRHWAI= -\n", + }, + } { + reader.Reset(test.input) + require.NoError(t, operations.HashSumStream(test.ht, false, in, out)) + assert.Equal(t, test.wantHex, out.String()) + _, _ = reader.Seek(0, io.SeekStart) + out.Reset() + require.NoError(t, operations.HashSumStream(test.ht, true, in, out)) + assert.Equal(t, test.wantBase64, out.String()) + out.Reset() + } +} + +func TestSuffixName(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + for _, test := range []struct { + remote string + suffix string + keepExt bool + want string + }{ + {"test.txt", "", false, "test.txt"}, + {"test.txt", "", true, "test.txt"}, + {"test.txt", "-suffix", false, "test.txt-suffix"}, + {"test.txt", "-suffix", true, "test-suffix.txt"}, + {"test.txt.csv", "-suffix", false, "test.txt.csv-suffix"}, + {"test.txt.csv", "-suffix", true, "test-suffix.txt.csv"}, + {"test", "-suffix", false, "test-suffix"}, + {"test", "-suffix", true, "test-suffix"}, + {"test.html", "-suffix", true, "test-suffix.html"}, + {"test.html.txt", "-suffix", true, "test-suffix.html.txt"}, + {"test.csv.html.txt", "-suffix", true, "test-suffix.csv.html.txt"}, + {"test.badext.csv.html.txt", "-suffix", true, "test.badext-suffix.csv.html.txt"}, + {"test.badext", "-suffix", true, "test-suffix.badext"}, + } { + ci.Suffix = test.suffix + ci.SuffixKeepExtension = test.keepExt + got := operations.SuffixName(ctx, test.remote) + assert.Equal(t, test.want, got, fmt.Sprintf("%+v", test)) + } +} + +func TestCount(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) + file2 := r.WriteBoth(ctx, "empty space", "-", t2) + file3 := r.WriteBoth(ctx, "sub dir/potato3", "hello", t2) + + r.CheckRemoteItems(t, file1, file2, file3) + + // Check the MaxDepth too + ci.MaxDepth = 1 + + objects, size, sizeless, err := operations.Count(ctx, r.Fremote) + require.NoError(t, err) + assert.Equal(t, int64(2), objects) + assert.Equal(t, int64(61), size) + assert.Equal(t, int64(0), sizeless) +} + +func TestDelete(t *testing.T) { + ctx := context.Background() + fi, err := filter.NewFilter(nil) + require.NoError(t, err) + fi.Opt.MaxSize = 60 + ctx = filter.ReplaceConfig(ctx, fi) + r := fstest.NewRun(t) + file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes + file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes + file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes + r.CheckRemoteItems(t, file1, file2, file3) + + err = operations.Delete(ctx, r.Fremote) + require.NoError(t, err) + r.CheckRemoteItems(t, file3) +} + +func isChunker(f fs.Fs) bool { + return strings.HasPrefix(f.Name(), "TestChunker") +} + +func skipIfChunker(t *testing.T, f fs.Fs) { + if isChunker(f) { + t.Skip("Skipping test on chunker backend") + } +} + +func TestMaxDelete(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + accounting.GlobalStats().ResetCounters() + ci.MaxDelete = 2 + defer r.Finalise() + skipIfChunker(t, r.Fremote) // chunker does copy/delete on s3 + file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes + file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes + file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes + r.CheckRemoteItems(t, file1, file2, file3) + err := operations.Delete(ctx, r.Fremote) + + require.Error(t, err) + objects, _, _, err := operations.Count(ctx, r.Fremote) + require.NoError(t, err) + assert.Equal(t, int64(1), objects) +} + +// TestMaxDeleteSizeLargeFile one of the files is larger than allowed +func TestMaxDeleteSizeLargeFile(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + accounting.GlobalStats().ResetCounters() + ci.MaxDeleteSize = 70 + defer r.Finalise() + skipIfChunker(t, r.Fremote) // chunker does copy/delete on s3 + file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes + file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes + file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes + r.CheckRemoteItems(t, file1, file2, file3) + + err := operations.Delete(ctx, r.Fremote) + require.Error(t, err) + r.CheckRemoteItems(t, file3) +} + +func TestMaxDeleteSize(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + accounting.GlobalStats().ResetCounters() + ci.MaxDeleteSize = 160 + defer r.Finalise() + skipIfChunker(t, r.Fremote) // chunker does copy/delete on s3 + file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes + file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes + file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes + r.CheckRemoteItems(t, file1, file2, file3) + + err := operations.Delete(ctx, r.Fremote) + require.Error(t, err) + objects, _, _, err := operations.Count(ctx, r.Fremote) + require.NoError(t, err) + assert.Equal(t, int64(1), objects) // 10 or 100 bytes +} + +func TestReadFile(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + defer r.Finalise() + + contents := "A file to read the contents." + file := r.WriteObject(ctx, "ReadFile", contents, t1) + r.CheckRemoteItems(t, file) + + o, err := r.Fremote.NewObject(ctx, file.Path) + require.NoError(t, err) + + buf, err := operations.ReadFile(ctx, o) + require.NoError(t, err) + assert.Equal(t, contents, string(buf)) +} + +func TestRetry(t *testing.T) { + ctx := context.Background() + + var i int + var err error + fn := func() error { + i-- + if i <= 0 { + return nil + } + return err + } + + i, err = 3, fmt.Errorf("Wrapped EOF is retriable: %w", io.EOF) + assert.Equal(t, nil, operations.Retry(ctx, nil, 5, fn)) + assert.Equal(t, 0, i) + + i, err = 10, pacer.RetryAfterError(errors.New("BANG"), 10*time.Millisecond) + assert.Equal(t, err, operations.Retry(ctx, nil, 5, fn)) + assert.Equal(t, 5, i) + + i, err = 10, fs.ErrorObjectNotFound + assert.Equal(t, fs.ErrorObjectNotFound, operations.Retry(ctx, nil, 5, fn)) + assert.Equal(t, 9, i) + +} + +func TestCat(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteBoth(ctx, "file1", "ABCDEFGHIJ", t1) + file2 := r.WriteBoth(ctx, "file2", "012345678", t2) + + r.CheckRemoteItems(t, file1, file2) + + for _, test := range []struct { + offset int64 + count int64 + separator string + a string + b string + }{ + {0, -1, "", "ABCDEFGHIJ", "012345678"}, + {0, 5, "", "ABCDE", "01234"}, + {-3, -1, "", "HIJ", "678"}, + {1, 3, "", "BCD", "123"}, + {0, -1, "\n", "ABCDEFGHIJ", "012345678"}, + } { + var buf bytes.Buffer + err := operations.Cat(ctx, r.Fremote, &buf, test.offset, test.count, []byte(test.separator)) + require.NoError(t, err) + res := buf.String() + + if res != test.a+test.separator+test.b+test.separator && res != test.b+test.separator+test.a+test.separator { + t.Errorf("Incorrect output from Cat(%d,%d,%s): %q", test.offset, test.count, test.separator, res) + } + } +} + +func TestPurge(t *testing.T) { + ctx := context.Background() + r := fstest.NewRunIndividual(t) // make new container (azureblob has delayed mkdir after rmdir) + r.Mkdir(ctx, r.Fremote) + + // Make some files and dirs + r.ForceMkdir(ctx, r.Fremote) + file1 := r.WriteObject(ctx, "A1/B1/C1/one", "aaa", t1) + //..and dirs we expect to delete + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A2")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2/C2")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C3")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3/B3")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3/B3/C4")) + //..and one more file at the end + file2 := r.WriteObject(ctx, "A1/two", "bbb", t2) + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + []fstest.Item{ + file1, file2, + }, + []string{ + "A1", + "A1/B1", + "A1/B1/C1", + "A2", + "A1/B2", + "A1/B2/C2", + "A1/B1/C3", + "A3", + "A3/B3", + "A3/B3/C4", + }, + fs.GetModifyWindow(ctx, r.Fremote), + ) + + require.NoError(t, operations.Purge(ctx, r.Fremote, "A1/B1")) + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + []fstest.Item{ + file2, + }, + []string{ + "A1", + "A2", + "A1/B2", + "A1/B2/C2", + "A3", + "A3/B3", + "A3/B3/C4", + }, + fs.GetModifyWindow(ctx, r.Fremote), + ) + + require.NoError(t, operations.Purge(ctx, r.Fremote, "")) + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + []fstest.Item{}, + []string{}, + fs.GetModifyWindow(ctx, r.Fremote), + ) + +} + +func TestRmdirsNoLeaveRoot(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + r.Mkdir(ctx, r.Fremote) + + // Make some files and dirs we expect to keep + r.ForceMkdir(ctx, r.Fremote) + file1 := r.WriteObject(ctx, "A1/B1/C1/one", "aaa", t1) + //..and dirs we expect to delete + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A2")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2/C2")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C3")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3/B3")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A3/B3/C4")) + //..and one more file at the end + file2 := r.WriteObject(ctx, "A1/two", "bbb", t2) + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + []fstest.Item{ + file1, file2, + }, + []string{ + "A1", + "A1/B1", + "A1/B1/C1", + "A2", + "A1/B2", + "A1/B2/C2", + "A1/B1/C3", + "A3", + "A3/B3", + "A3/B3/C4", + }, + fs.GetModifyWindow(ctx, r.Fremote), + ) + + require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "A3/B3/C4", false)) + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + []fstest.Item{ + file1, file2, + }, + []string{ + "A1", + "A1/B1", + "A1/B1/C1", + "A2", + "A1/B2", + "A1/B2/C2", + "A1/B1/C3", + "A3", + "A3/B3", + }, + fs.GetModifyWindow(ctx, r.Fremote), + ) + + require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "", false)) + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + []fstest.Item{ + file1, file2, + }, + []string{ + "A1", + "A1/B1", + "A1/B1/C1", + }, + fs.GetModifyWindow(ctx, r.Fremote), + ) + + // Delete the files so we can remove everything including the root + for _, file := range []fstest.Item{file1, file2} { + o, err := r.Fremote.NewObject(ctx, file.Path) + require.NoError(t, err) + require.NoError(t, o.Remove(ctx)) + } + + require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "", false)) + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + []fstest.Item{}, + []string{}, + fs.GetModifyWindow(ctx, r.Fremote), + ) +} + +func TestRmdirsLeaveRoot(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + r.Mkdir(ctx, r.Fremote) + + r.ForceMkdir(ctx, r.Fremote) + + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C1")) + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + []fstest.Item{}, + []string{ + "A1", + "A1/B1", + "A1/B1/C1", + }, + fs.GetModifyWindow(ctx, r.Fremote), + ) + + require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "A1", true)) + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + []fstest.Item{}, + []string{ + "A1", + }, + fs.GetModifyWindow(ctx, r.Fremote), + ) +} + +func TestRmdirsWithFilter(t *testing.T) { + ctx := context.Background() + ctx, fi := filter.AddConfig(ctx) + require.NoError(t, fi.AddRule("+ /A1/B1/**")) + require.NoError(t, fi.AddRule("- *")) + r := fstest.NewRun(t) + r.Mkdir(ctx, r.Fremote) + + r.ForceMkdir(ctx, r.Fremote) + + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C1")) + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + []fstest.Item{}, + []string{ + "A1", + "A1/B1", + "A1/B1/C1", + }, + fs.GetModifyWindow(ctx, r.Fremote), + ) + + require.NoError(t, operations.Rmdirs(ctx, r.Fremote, "", false)) + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + []fstest.Item{}, + []string{ + "A1", + }, + fs.GetModifyWindow(ctx, r.Fremote), + ) +} + +func TestCopyURL(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + contents := "file contents\n" + file1 := r.WriteFile("file1", contents, t1) + file2 := r.WriteFile("file2", contents, t1) + r.Mkdir(ctx, r.Fremote) + r.CheckRemoteItems(t) + + // check when reading from regular HTTP server + status := 0 + nameHeader := false + headerFilename := "headerfilename.txt" + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if status != 0 { + http.Error(w, "an error occurred", status) + } + if nameHeader { + w.Header().Set("Content-Disposition", `attachment; filename="folder\`+headerFilename+`"`) + } + _, err := w.Write([]byte(contents)) + assert.NoError(t, err) + }) + ts := httptest.NewServer(handler) + defer ts.Close() + + o, err := operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, false) + require.NoError(t, err) + assert.Equal(t, int64(len(contents)), o.Size()) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported) + + // Check file clobbering + _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, true) + require.Error(t, err) + + // Check auto file naming + status = 0 + urlFileName := "filename.txt" + o, err = operations.CopyURL(ctx, r.Fremote, "", ts.URL+"/"+urlFileName, true, false, false) + require.NoError(t, err) + assert.Equal(t, int64(len(contents)), o.Size()) + assert.Equal(t, urlFileName, o.Remote()) + + // Check header file naming + nameHeader = true + o, err = operations.CopyURL(ctx, r.Fremote, "", ts.URL, true, true, false) + require.NoError(t, err) + assert.Equal(t, int64(len(contents)), o.Size()) + assert.Equal(t, headerFilename, o.Remote()) + + // Check auto file naming when url without file name + _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, true, false, false) + require.Error(t, err) + + // Check header file naming without header set + nameHeader = false + _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, true, true, false) + require.Error(t, err) + + // Check an error is returned for a 404 + status = http.StatusNotFound + o, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, false) + require.Error(t, err) + assert.Contains(t, err.Error(), "Not Found") + assert.Nil(t, o) + status = 0 + + // check when reading from unverified HTTPS server + ci.InsecureSkipVerify = true + fshttp.ResetTransport() + defer fshttp.ResetTransport() + tss := httptest.NewTLSServer(handler) + defer tss.Close() + + o, err = operations.CopyURL(ctx, r.Fremote, "file2", tss.URL, false, false, false) + require.NoError(t, err) + assert.Equal(t, int64(len(contents)), o.Size()) + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1), fstest.NewItem(headerFilename, contents, t1)}, nil, fs.ModTimeNotSupported) +} + +func TestCopyURLToWriter(t *testing.T) { + ctx := context.Background() + contents := "file contents\n" + + // check when reading from regular HTTP server + status := 0 + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if status != 0 { + http.Error(w, "an error occurred", status) + return + } + _, err := w.Write([]byte(contents)) + assert.NoError(t, err) + }) + ts := httptest.NewServer(handler) + defer ts.Close() + + // test normal fetch + var buf bytes.Buffer + err := operations.CopyURLToWriter(ctx, ts.URL, &buf) + require.NoError(t, err) + assert.Equal(t, contents, buf.String()) + + // test fetch with error + status = http.StatusNotFound + buf.Reset() + err = operations.CopyURLToWriter(ctx, ts.URL, &buf) + require.Error(t, err) + assert.Contains(t, err.Error(), "Not Found") + assert.Equal(t, 0, len(buf.String())) +} + +func TestMoveFile(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + + file1 := r.WriteFile("file1", "file1 contents", t1) + r.CheckLocalItems(t, file1) + + file2 := file1 + file2.Path = "sub/file2" + + err := operations.MoveFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t) + r.CheckRemoteItems(t, file2) + + r.WriteFile("file1", "file1 contents", t1) + r.CheckLocalItems(t, file1) + + err = operations.MoveFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t) + r.CheckRemoteItems(t, file2) + + err = operations.MoveFile(ctx, r.Fremote, r.Fremote, file2.Path, file2.Path) + require.NoError(t, err) + r.CheckLocalItems(t) + r.CheckRemoteItems(t, file2) +} + +func TestMoveFileWithIgnoreExisting(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + file1 := r.WriteFile("file1", "file1 contents", t1) + r.CheckLocalItems(t, file1) + + ci.IgnoreExisting = true + + err := operations.MoveFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t) + r.CheckRemoteItems(t, file1) + + // Recreate file with updated content + file1b := r.WriteFile("file1", "file1 modified", t2) + r.CheckLocalItems(t, file1b) + + // Ensure modified file did not transfer and was not deleted + err = operations.MoveFile(ctx, r.Fremote, r.Flocal, file1.Path, file1b.Path) + require.NoError(t, err) + r.CheckLocalItems(t, file1b) + r.CheckRemoteItems(t, file1) +} + +func TestCaseInsensitiveMoveFile(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + if !r.Fremote.Features().CaseInsensitive { + return + } + + file1 := r.WriteFile("file1", "file1 contents", t1) + r.CheckLocalItems(t, file1) + + file2 := file1 + file2.Path = "sub/file2" + + err := operations.MoveFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t) + r.CheckRemoteItems(t, file2) + + r.WriteFile("file1", "file1 contents", t1) + r.CheckLocalItems(t, file1) + + err = operations.MoveFile(ctx, r.Fremote, r.Flocal, file2.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t) + r.CheckRemoteItems(t, file2) + + file2Capitalized := file2 + file2Capitalized.Path = "sub/File2" + + err = operations.MoveFile(ctx, r.Fremote, r.Fremote, file2Capitalized.Path, file2.Path) + require.NoError(t, err) + r.CheckLocalItems(t) + r.CheckRemoteItems(t, file2Capitalized) +} + +func TestCaseInsensitiveMoveFileDryRun(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + if !r.Fremote.Features().CaseInsensitive { + return + } + + file1 := r.WriteObject(ctx, "hello", "world", t1) + r.CheckRemoteItems(t, file1) + + ci.DryRun = true + err := operations.MoveFile(ctx, r.Fremote, r.Fremote, "HELLO", file1.Path) + require.NoError(t, err) + r.CheckRemoteItems(t, file1) +} + +func TestMoveFileBackupDir(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + if !operations.CanServerSideMove(r.Fremote) { + t.Skip("Skipping test as remote does not support server-side move or copy") + } + + ci.BackupDir = r.FremoteName + "/backup" + + file1 := r.WriteFile("dst/file1", "file1 contents", t1) + r.CheckLocalItems(t, file1) + + file1old := r.WriteObject(ctx, "dst/file1", "file1 contents old", t1) + r.CheckRemoteItems(t, file1old) + + err := operations.MoveFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path) + require.NoError(t, err) + r.CheckLocalItems(t) + file1old.Path = "backup/dst/file1" + r.CheckRemoteItems(t, file1old, file1) +} + +// testFsInfo is for unit testing fs.Info +type testFsInfo struct { + name string + root string + stringVal string + precision time.Duration + hashes hash.Set + features fs.Features +} + +// Name of the remote (as passed into NewFs) +func (i *testFsInfo) Name() string { return i.name } + +// Root of the remote (as passed into NewFs) +func (i *testFsInfo) Root() string { return i.root } + +// String returns a description of the FS +func (i *testFsInfo) String() string { return i.stringVal } + +// Precision of the ModTimes in this Fs +func (i *testFsInfo) Precision() time.Duration { return i.precision } + +// Returns the supported hash types of the filesystem +func (i *testFsInfo) Hashes() hash.Set { return i.hashes } + +// Returns the supported hash types of the filesystem +func (i *testFsInfo) Features() *fs.Features { return &i.features } + +func TestSameConfig(t *testing.T) { + a := &testFsInfo{name: "name", root: "root"} + for _, test := range []struct { + name string + root string + expected bool + }{ + {"name", "root", true}, + {"name", "rooty", true}, + {"namey", "root", false}, + {"namey", "roott", false}, + } { + b := &testFsInfo{name: test.name, root: test.root} + actual := operations.SameConfig(a, b) + assert.Equal(t, test.expected, actual) + actual = operations.SameConfig(b, a) + assert.Equal(t, test.expected, actual) + } +} + +func TestSame(t *testing.T) { + a := &testFsInfo{name: "name", root: "root"} + for _, test := range []struct { + name string + root string + expected bool + }{ + {"name", "root", true}, + {"name", "rooty", false}, + {"namey", "root", false}, + {"namey", "roott", false}, + } { + b := &testFsInfo{name: test.name, root: test.root} + actual := operations.Same(a, b) + assert.Equal(t, test.expected, actual) + actual = operations.Same(b, a) + assert.Equal(t, test.expected, actual) + } +} + +// testFs is for unit testing fs.Fs +type testFs struct { + testFsInfo +} + +func (i *testFs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + return nil, nil +} + +func (i *testFs) NewObject(ctx context.Context, remote string) (fs.Object, error) { return nil, nil } + +func (i *testFs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return nil, nil +} + +func (i *testFs) Mkdir(ctx context.Context, dir string) error { return nil } + +func (i *testFs) Rmdir(ctx context.Context, dir string) error { return nil } + +// copied from TestOverlapping because the behavior of OverlappingFilterCheck should be identical to Overlapping +// when no filters are set +func TestOverlappingFilterCheckWithoutFilter(t *testing.T) { + ctx := context.Background() + src := &testFs{testFsInfo{name: "name", root: "root"}} + slash := string(os.PathSeparator) // native path separator + for _, test := range []struct { + name string + root string + expected bool + }{ + {"name", "root", true}, + {"name", "/root", true}, + {"namey", "root", false}, + {"name", "rooty", false}, + {"namey", "rooty", false}, + {"name", "roo", false}, + {"name", "root/toot", true}, + {"name", "root/toot/", true}, + {"name", "root" + slash + "toot", true}, + {"name", "root" + slash + "toot" + slash, true}, + {"name", "", true}, + {"name", "/", true}, + } { + dst := &testFs{testFsInfo{name: test.name, root: test.root}} + what := fmt.Sprintf("(%q,%q) vs (%q,%q)", src.name, src.root, dst.name, dst.root) + actual := operations.OverlappingFilterCheck(ctx, src, dst) + assert.Equal(t, test.expected, actual, what) + actual = operations.OverlappingFilterCheck(ctx, dst, src) + assert.Equal(t, test.expected, actual, what) + } +} + +func TestOverlappingFilterCheckWithFilter(t *testing.T) { + ctx := context.Background() + fi, err := filter.NewFilter(nil) + require.NoError(t, err) + require.NoError(t, fi.Add(false, "/exclude/")) + require.NoError(t, fi.Add(false, "/Exclude2/")) + require.NoError(t, fi.Add(true, "*")) + ctx = filter.ReplaceConfig(ctx, fi) + + src := &testFs{testFsInfo{name: "name", root: "root"}} + src.features.CaseInsensitive = true + slash := string(os.PathSeparator) // native path separator + for _, test := range []struct { + name string + root string + expected bool + }{ + {"name", "root", true}, + {"name", "ROOT", true}, // case insensitive is set + {"name", "/root", true}, + {"name", "root/", true}, + {"name", "root" + slash, true}, + {"name", "root/exclude", false}, + {"name", "root/Exclude2", false}, + {"name", "root/include", true}, + {"name", "root/exclude/", false}, + {"name", "root/Exclude2/", false}, + {"name", "root/exclude/sub", false}, + {"name", "root/Exclude2/sub", false}, + {"name", "/root/exclude/", false}, + {"name", "root" + slash + "exclude", false}, + {"name", "root" + slash + "exclude" + slash, false}, + {"namey", "root/include", false}, + {"namey", "root/include/", false}, + {"namey", "root" + slash + "include", false}, + {"namey", "root" + slash + "include" + slash, false}, + } { + dst := &testFs{testFsInfo{name: test.name, root: test.root}} + dst.features.CaseInsensitive = true + what := fmt.Sprintf("(%q,%q) vs (%q,%q)", src.name, src.root, dst.name, dst.root) + actual := operations.OverlappingFilterCheck(ctx, dst, src) + assert.Equal(t, test.expected, actual, what) + actual = operations.OverlappingFilterCheck(ctx, src, dst) + assert.Equal(t, test.expected, actual, what) + } +} + +func TestListFormat(t *testing.T) { + item0 := &operations.ListJSONItem{ + Path: "a", + Name: "a", + Encrypted: "encryptedFileName", + Size: 1, + MimeType: "application/octet-stream", + ModTime: operations.Timestamp{ + When: t1, + Format: "2006-01-02T15:04:05.000000000Z07:00"}, + IsDir: false, + Hashes: map[string]string{ + "md5": "0cc175b9c0f1b6a831c399e269772661", + "sha1": "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", + "dropbox": "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8", + "quickxor": "6100000000000000000000000100000000000000"}, + ID: "fileID", + OrigID: "fileOrigID", + } + + item1 := &operations.ListJSONItem{ + Path: "subdir", + Name: "subdir", + Encrypted: "encryptedDirName", + Size: -1, + MimeType: "inode/directory", + ModTime: operations.Timestamp{ + When: t2, + Format: "2006-01-02T15:04:05.000000000Z07:00"}, + IsDir: true, + Hashes: map[string]string(nil), + ID: "dirID", + OrigID: "dirOrigID", + } + + var list operations.ListFormat + list.AddPath() + list.SetDirSlash(false) + assert.Equal(t, "subdir", list.Format(item1)) + + list.SetDirSlash(true) + assert.Equal(t, "subdir/", list.Format(item1)) + + list.SetOutput(nil) + assert.Equal(t, "", list.Format(item1)) + + list.AppendOutput(func(item *operations.ListJSONItem) string { return "a" }) + list.AppendOutput(func(item *operations.ListJSONItem) string { return "b" }) + assert.Equal(t, "ab", list.Format(item1)) + list.SetSeparator(":::") + assert.Equal(t, "a:::b", list.Format(item1)) + + list.SetOutput(nil) + list.AddModTime("") + assert.Equal(t, t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) + + list.SetOutput(nil) + list.AddModTime("unix") + assert.Equal(t, fmt.Sprint(t1.Local().Unix()), list.Format(item0)) + + list.SetOutput(nil) + list.AddModTime("unixnano") + assert.Equal(t, fmt.Sprint(t1.Local().UnixNano()), list.Format(item0)) + + list.SetOutput(nil) + list.SetSeparator("|") + list.AddID() + list.AddOrigID() + assert.Equal(t, "fileID|fileOrigID", list.Format(item0)) + assert.Equal(t, "dirID|dirOrigID", list.Format(item1)) + + list.SetOutput(nil) + list.AddMimeType() + assert.Contains(t, list.Format(item0), "/") + assert.Equal(t, "inode/directory", list.Format(item1)) + + list.SetOutput(nil) + list.AddMetadata() + assert.Equal(t, "{}", list.Format(item0)) + assert.Equal(t, "{}", list.Format(item1)) + + list.SetOutput(nil) + list.AddPath() + list.SetAbsolute(true) + assert.Equal(t, "/a", list.Format(item0)) + list.SetAbsolute(false) + assert.Equal(t, "a", list.Format(item0)) + + list.SetOutput(nil) + list.AddSize() + assert.Equal(t, "1", list.Format(item0)) + + list.AddPath() + list.AddModTime("") + list.SetDirSlash(true) + list.SetSeparator("__SEP__") + assert.Equal(t, "1__SEP__a__SEP__"+t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) + assert.Equal(t, "-1__SEP__subdir/__SEP__"+t2.Local().Format("2006-01-02 15:04:05"), list.Format(item1)) + + for _, test := range []struct { + ht hash.Type + want string + }{ + {hash.MD5, "0cc175b9c0f1b6a831c399e269772661"}, + {hash.SHA1, "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8"}, + } { + list.SetOutput(nil) + list.AddHash(test.ht) + assert.Equal(t, test.want, list.Format(item0)) + } + + list.SetOutput(nil) + list.SetSeparator("|") + list.SetCSV(true) + list.AddSize() + list.AddPath() + list.AddModTime("") + list.SetDirSlash(true) + assert.Equal(t, "1|a|"+t1.Local().Format("2006-01-02 15:04:05"), list.Format(item0)) + assert.Equal(t, "-1|subdir/|"+t2.Local().Format("2006-01-02 15:04:05"), list.Format(item1)) + + list.SetOutput(nil) + list.SetSeparator("|") + list.AddPath() + list.AddEncrypted() + assert.Equal(t, "a|encryptedFileName", list.Format(item0)) + assert.Equal(t, "subdir/|encryptedDirName/", list.Format(item1)) + +} + +func TestDirMove(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + + r.Mkdir(ctx, r.Fremote) + + // Make some files and dirs + r.ForceMkdir(ctx, r.Fremote) + files := []fstest.Item{ + r.WriteObject(ctx, "A1/one", "one", t1), + r.WriteObject(ctx, "A1/two", "two", t2), + r.WriteObject(ctx, "A1/B1/three", "three", t3), + r.WriteObject(ctx, "A1/B1/C1/four", "four", t1), + r.WriteObject(ctx, "A1/B1/C2/five", "five", t2), + } + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B2")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "A1/B1/C3")) + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + files, + []string{ + "A1", + "A1/B1", + "A1/B2", + "A1/B1/C1", + "A1/B1/C2", + "A1/B1/C3", + }, + fs.GetModifyWindow(ctx, r.Fremote), + ) + + require.NoError(t, operations.DirMove(ctx, r.Fremote, "A1", "A2")) + + for i := range files { + files[i].Path = strings.ReplaceAll(files[i].Path, "A1/", "A2/") + } + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + files, + []string{ + "A2", + "A2/B1", + "A2/B2", + "A2/B1/C1", + "A2/B1/C2", + "A2/B1/C3", + }, + fs.GetModifyWindow(ctx, r.Fremote), + ) + + // Disable DirMove + features := r.Fremote.Features() + features.DirMove = nil + + require.NoError(t, operations.DirMove(ctx, r.Fremote, "A2", "A3")) + + for i := range files { + files[i].Path = strings.ReplaceAll(files[i].Path, "A2/", "A3/") + } + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + files, + []string{ + "A3", + "A3/B1", + "A3/B2", + "A3/B1/C1", + "A3/B1/C2", + "A3/B1/C3", + }, + fs.GetModifyWindow(ctx, r.Fremote), + ) + + // Try with a DirMove method that exists but returns fs.ErrorCantDirMove (ex. combine moving across upstreams) + // Should fall back to manual move (copy + delete) + + features.DirMove = func(ctx context.Context, src fs.Fs, srcRemote string, dstRemote string) error { + return fs.ErrorCantDirMove + } + + assert.NoError(t, operations.DirMove(ctx, r.Fremote, "A3", "A4")) + + for i := range files { + files[i].Path = strings.ReplaceAll(files[i].Path, "A3/", "A4/") + } + + fstest.CheckListingWithPrecision( + t, + r.Fremote, + files, + []string{ + "A4", + "A4/B1", + "A4/B2", + "A4/B1/C1", + "A4/B1/C2", + "A4/B1/C3", + }, + fs.GetModifyWindow(ctx, r.Fremote), + ) +} + +func TestGetFsInfo(t *testing.T) { + r := fstest.NewRun(t) + + f := r.Fremote + info := operations.GetFsInfo(f) + assert.Equal(t, f.Name(), info.Name) + assert.Equal(t, f.Root(), info.Root) + assert.Equal(t, f.String(), info.String) + assert.Equal(t, f.Precision(), info.Precision) + hashSet := hash.NewHashSet() + for _, hashName := range info.Hashes { + var ht hash.Type + require.NoError(t, ht.Set(hashName)) + hashSet.Add(ht) + } + assert.Equal(t, f.Hashes(), hashSet) + assert.Equal(t, f.Features().Enabled(), info.Features) +} + +func TestRcat(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + check := func(t *testing.T, withChecksum, ignoreChecksum bool) { + ci.CheckSum, ci.IgnoreChecksum = withChecksum, ignoreChecksum + + var prefix string + if withChecksum { + prefix = "with_checksum_" + } else { + prefix = "no_checksum_" + } + if ignoreChecksum { + prefix = "ignore_checksum_" + } + + r := fstest.NewRun(t) + + if *fstest.SizeLimit > 0 && int64(ci.StreamingUploadCutoff) > *fstest.SizeLimit { + savedCutoff := ci.StreamingUploadCutoff + ci.StreamingUploadCutoff = fs.SizeSuffix(*fstest.SizeLimit) + t.Logf("Adjust StreamingUploadCutoff to size limit %s (was %s)", ci.StreamingUploadCutoff, savedCutoff) + } + + fstest.CheckListing(t, r.Fremote, []fstest.Item{}) + + data1 := "this is some really nice test data" + path1 := prefix + "small_file_from_pipe" + + data2 := string(make([]byte, ci.StreamingUploadCutoff+1)) + path2 := prefix + "big_file_from_pipe" + + in := io.NopCloser(strings.NewReader(data1)) + _, err := operations.Rcat(ctx, r.Fremote, path1, in, t1, nil) + require.NoError(t, err) + + in = io.NopCloser(strings.NewReader(data2)) + _, err = operations.Rcat(ctx, r.Fremote, path2, in, t2, nil) + require.NoError(t, err) + + file1 := fstest.NewItem(path1, data1, t1) + file2 := fstest.NewItem(path2, data2, t2) + r.CheckRemoteItems(t, file1, file2) + } + + for i := range 4 { + withChecksum := (i & 1) != 0 + ignoreChecksum := (i & 2) != 0 + t.Run(fmt.Sprintf("withChecksum=%v,ignoreChecksum=%v", withChecksum, ignoreChecksum), func(t *testing.T) { + check(t, withChecksum, ignoreChecksum) + }) + } +} + +func TestRcatMetadata(t *testing.T) { + r := fstest.NewRun(t) + + if !r.Fremote.Features().UserMetadata { + t.Skip("Skipping as destination doesn't support user metadata") + } + + test := func(disableUploadCutoff bool) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + data := "this is some really nice test data with metadata" + path := "rcat_metadata" + + meta := fs.Metadata{ + "key": "value", + "sausage": "potato", + } + + if disableUploadCutoff { + ci.StreamingUploadCutoff = 0 + data += " uploadCutoff=0" + path += "_uploadcutoff0" + } + + fstest.CheckListing(t, r.Fremote, []fstest.Item{}) + + in := io.NopCloser(strings.NewReader(data)) + _, err := operations.Rcat(ctx, r.Fremote, path, in, t1, meta) + require.NoError(t, err) + + file := fstest.NewItem(path, data, t1) + r.CheckRemoteItems(t, file) + + o, err := r.Fremote.NewObject(ctx, path) + require.NoError(t, err) + gotMeta, err := fs.GetMetadata(ctx, o) + require.NoError(t, err) + // Check the specific user data we set is set + // Likely there will be other values + assert.Equal(t, "value", gotMeta["key"]) + assert.Equal(t, "potato", gotMeta["sausage"]) + + // Delete the test file + require.NoError(t, o.Remove(ctx)) + } + + t.Run("Normal", func(t *testing.T) { + test(false) + }) + t.Run("ViaDisk", func(t *testing.T) { + test(true) + }) +} + +func TestRcatSize(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + + const body = "------------------------------------------------------------" + file1 := r.WriteFile("potato1", body, t1) + file2 := r.WriteFile("potato2", body, t2) + // Test with known length + bodyReader := io.NopCloser(strings.NewReader(body)) + obj, err := operations.RcatSize(ctx, r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime, nil) + require.NoError(t, err) + assert.Equal(t, int64(len(body)), obj.Size()) + assert.Equal(t, file1.Path, obj.Remote()) + + // Test with unknown length + bodyReader = io.NopCloser(strings.NewReader(body)) // reset Reader + io.NopCloser(strings.NewReader(body)) + obj, err = operations.RcatSize(ctx, r.Fremote, file2.Path, bodyReader, -1, file2.ModTime, nil) + require.NoError(t, err) + assert.Equal(t, int64(len(body)), obj.Size()) + assert.Equal(t, file2.Path, obj.Remote()) + + // Check files exist + r.CheckRemoteItems(t, file1, file2) +} + +func TestRcatSizeMetadata(t *testing.T) { + r := fstest.NewRun(t) + + if !r.Fremote.Features().UserMetadata { + t.Skip("Skipping as destination doesn't support user metadata") + } + + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + + meta := fs.Metadata{ + "key": "value", + "sausage": "potato", + } + + const body = "------------------------------------------------------------" + file1 := r.WriteFile("potato1", body, t1) + file2 := r.WriteFile("potato2", body, t2) + + // Test with known length + bodyReader := io.NopCloser(strings.NewReader(body)) + obj, err := operations.RcatSize(ctx, r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime, meta) + require.NoError(t, err) + assert.Equal(t, int64(len(body)), obj.Size()) + assert.Equal(t, file1.Path, obj.Remote()) + + // Test with unknown length + bodyReader = io.NopCloser(strings.NewReader(body)) // reset Reader + io.NopCloser(strings.NewReader(body)) + obj, err = operations.RcatSize(ctx, r.Fremote, file2.Path, bodyReader, -1, file2.ModTime, meta) + require.NoError(t, err) + assert.Equal(t, int64(len(body)), obj.Size()) + assert.Equal(t, file2.Path, obj.Remote()) + + // Check files exist + r.CheckRemoteItems(t, file1, file2) + + // Check metadata OK + for _, path := range []string{file1.Path, file2.Path} { + o, err := r.Fremote.NewObject(ctx, path) + require.NoError(t, err) + gotMeta, err := fs.GetMetadata(ctx, o) + require.NoError(t, err) + // Check the specific user data we set is set + // Likely there will be other values + assert.Equal(t, "value", gotMeta["key"]) + assert.Equal(t, "potato", gotMeta["sausage"]) + } +} + +func TestTouchDir(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + + if r.Fremote.Precision() == fs.ModTimeNotSupported { + t.Skip("Skipping test as remote does not support modtime") + } + + file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) + file2 := r.WriteBoth(ctx, "empty space", "-", t2) + file3 := r.WriteBoth(ctx, "sub dir/potato3", "hello", t2) + r.CheckRemoteItems(t, file1, file2, file3) + + accounting.GlobalStats().ResetCounters() + timeValue := time.Date(2010, 9, 8, 7, 6, 5, 4, time.UTC) + err := operations.TouchDir(ctx, r.Fremote, "", timeValue, true) + require.NoError(t, err) + if accounting.Stats(ctx).GetErrors() != 0 { + err = accounting.Stats(ctx).GetLastError() + require.True(t, errors.Is(err, fs.ErrorCantSetModTime) || errors.Is(err, fs.ErrorCantSetModTimeWithoutDelete)) + } else { + file1.ModTime = timeValue + file2.ModTime = timeValue + file3.ModTime = timeValue + r.CheckRemoteItems(t, file1, file2, file3) + } +} + +var testMetadata = fs.Metadata{ + // System metadata supported by all backends + "mtime": t1.Format(time.RFC3339Nano), + // User metadata + "potato": "jersey", +} + +func TestMkdirMetadata(t *testing.T) { + const name = "dir with metadata" + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + r := fstest.NewRun(t) + features := r.Fremote.Features() + if features.MkdirMetadata == nil { + t.Skip("Skipping test as remote does not support MkdirMetadata") + } + + newDst, err := operations.MkdirMetadata(ctx, r.Fremote, name, testMetadata) + require.NoError(t, err) + require.NotNil(t, newDst) + + require.True(t, features.ReadDirMetadata, "Expecting ReadDirMetadata to be supported if MkdirMetadata is supported") + + // Check the returned directory and one read from the listing + fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata) + fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), testMetadata) +} + +func TestMkdirModTime(t *testing.T) { + const name = "directory with modtime" + ctx := context.Background() + r := fstest.NewRun(t) + if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil { + t.Skip("Skipping test as remote does not support DirSetModTime or MkdirMetadata") + } + newDst, err := operations.MkdirModTime(ctx, r.Fremote, name, t2) + require.NoError(t, err) + + // Check the returned directory and one read from the listing + // newDst may be nil here depending on how the modtime was set + if newDst != nil { + fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2) + } + fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2) +} + +func TestCopyDirMetadata(t *testing.T) { + const nameNonExistent = "non existent directory" + const nameExistent = "existing directory" + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + r := fstest.NewRun(t) + if !r.Fremote.Features().WriteDirMetadata && r.Fremote.Features().MkdirMetadata == nil { + t.Skip("Skipping test as remote does not support WriteDirMetadata or MkdirMetadata") + } + + // Create a source local directory with metadata + newSrc, err := operations.MkdirMetadata(ctx, r.Flocal, "dir with metadata to be copied", testMetadata) + require.NoError(t, err) + require.NotNil(t, newSrc) + + // First try with the directory not existing + newDst, err := operations.CopyDirMetadata(ctx, r.Fremote, nil, nameNonExistent, newSrc) + require.NoError(t, err) + require.NotNil(t, newDst) + + // Check the returned directory and one read from the listing + fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata) + fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameNonExistent), testMetadata) + + // Then try with the directory existing + require.NoError(t, r.Fremote.Rmdir(ctx, nameNonExistent)) + require.NoError(t, r.Fremote.Mkdir(ctx, nameExistent)) + existingDir := fstest.NewDirectory(ctx, t, r.Fremote, nameExistent) + + newDst, err = operations.CopyDirMetadata(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", newSrc) + require.NoError(t, err) + require.NotNil(t, newDst) + + // Check the returned directory and one read from the listing + fstest.CheckEntryMetadata(ctx, t, r.Fremote, newDst, testMetadata) + fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, nameExistent), testMetadata) +} + +func TestSetDirModTime(t *testing.T) { + const name = "set modtime on existing directory" + ctx, ci := fs.AddConfig(context.Background()) + r := fstest.NewRun(t) + if r.Fremote.Features().DirSetModTime == nil && !r.Fremote.Features().WriteDirSetModTime { + t.Skip("Skipping test as remote does not support DirSetModTime or WriteDirSetModTime") + } + + // Check that we obey --no-update-dir-modtime - this should return nil, nil + ci.NoUpdateDirModTime = true + newDst, err := operations.SetDirModTime(ctx, r.Fremote, nil, "set modtime on non existent directory", t2) + require.NoError(t, err) + require.Nil(t, newDst) + ci.NoUpdateDirModTime = false + + // First try with the directory not existing - should return an error + newDst, err = operations.SetDirModTime(ctx, r.Fremote, nil, "set modtime on non existent directory", t2) + require.Error(t, err) + require.Nil(t, newDst) + + // Then try with the directory existing + require.NoError(t, r.Fremote.Mkdir(ctx, name)) + existingDir := fstest.NewDirectory(ctx, t, r.Fremote, name) + + newDst, err = operations.SetDirModTime(ctx, r.Fremote, existingDir, "SHOULD BE IGNORED", t2) + require.NoError(t, err) + require.NotNil(t, newDst) + + // Check the returned directory and one read from the listing + // The modtime will only be correct on newDst if it had a SetModTime method + if _, ok := newDst.(fs.SetModTimer); ok { + fstest.CheckDirModTime(ctx, t, r.Fremote, newDst, t2) + } + fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2) + + // Now wrap the directory to make the SetModTime method return fs.ErrorNotImplemented and check that it falls back correctly + wrappedDir := fs.NewDirWrapper(existingDir.Remote(), fs.NewDir(existingDir.Remote(), existingDir.ModTime(ctx))) + newDst, err = operations.SetDirModTime(ctx, r.Fremote, wrappedDir, "SHOULD BE IGNORED", t1) + require.NoError(t, err) + require.NotNil(t, newDst) + fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t1) +} + +func TestDirsEqual(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + r := fstest.NewRun(t) + if !r.Fremote.Features().WriteDirMetadata && r.Fremote.Features().MkdirMetadata == nil { + t.Skip("Skipping test as remote does not support WriteDirMetadata or MkdirMetadata") + } + + opt := operations.DirsEqualOpt{ + ModifyWindow: fs.GetModifyWindow(ctx, r.Flocal, r.Fremote), + SetDirModtime: true, + SetDirMetadata: true, + } + + // Create a source local directory with metadata + src, err := operations.MkdirMetadata(ctx, r.Flocal, "dir with metadata to be copied", testMetadata) + require.NoError(t, err) + require.NotNil(t, src) + + // try with nil dst -- should be false + equal := operations.DirsEqual(ctx, src, nil, opt) + assert.False(t, equal) + + // make a dest with an equal modtime + dst, err := operations.MkdirModTime(ctx, r.Fremote, "dst", src.ModTime(ctx)) + require.NoError(t, err) + + // try with equal modtimes -- should be true + equal = operations.DirsEqual(ctx, src, dst, opt) + assert.True(t, equal) + + // try with unequal modtimes -- should be false + dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", t2) + require.NoError(t, err) + equal = operations.DirsEqual(ctx, src, dst, opt) + assert.False(t, equal) + + // try with unequal modtimes that are within modify window -- should be true + halfWindow := opt.ModifyWindow / 2 + dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", src.ModTime(ctx).Add(halfWindow)) + require.NoError(t, err) + equal = operations.DirsEqual(ctx, src, dst, opt) + assert.True(t, equal) + + // test ignoretimes -- should be false + ci.IgnoreTimes = true + equal = operations.DirsEqual(ctx, src, dst, opt) + assert.False(t, equal) + + // test immutable -- should be true + ci.IgnoreTimes = false + ci.Immutable = true + dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", t3) + require.NoError(t, err) + equal = operations.DirsEqual(ctx, src, dst, opt) + assert.True(t, equal) + + // test dst newer than src with --update -- should be true + ci.Immutable = false + ci.UpdateOlder = true + equal = operations.DirsEqual(ctx, src, dst, opt) + assert.True(t, equal) + + // test no SetDirModtime or SetDirMetadata -- should be true + ci.UpdateOlder = false + opt.SetDirMetadata, opt.SetDirModtime = false, false + equal = operations.DirsEqual(ctx, src, dst, opt) + assert.True(t, equal) +} + +func TestRemoveExisting(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + if r.Fremote.Features().Move == nil { + t.Skip("Skipping as remote can't Move") + } + + file1 := r.WriteObject(ctx, "sub dir/test remove existing", "hello world", t1) + file2 := r.WriteObject(ctx, "sub dir/test remove existing with long name 123456789012345678901234567890123456789012345678901234567890123456789", "hello long name world", t1) + + r.CheckRemoteItems(t, file1, file2) + + var returnedError error + + // Check not found first + cleanup, err := operations.RemoveExisting(ctx, r.Fremote, "not found", "TEST") + assert.Equal(t, err, nil) + r.CheckRemoteItems(t, file1, file2) + cleanup(&returnedError) + r.CheckRemoteItems(t, file1, file2) + + // Remove file1 + cleanup, err = operations.RemoveExisting(ctx, r.Fremote, file1.Path, "TEST") + assert.Equal(t, err, nil) + //r.CheckRemoteItems(t, file1, file2) + + // Check file1 with temporary name exists + var buf bytes.Buffer + err = operations.List(ctx, r.Fremote, &buf) + require.NoError(t, err) + res := buf.String() + assert.NotContains(t, res, " 11 "+file1.Path+"\n") + assert.Contains(t, res, " 11 "+file1.Path+".") + assert.Contains(t, res, " 21 "+file2.Path+"\n") + + cleanup(&returnedError) + r.CheckRemoteItems(t, file2) + + // Remove file2 with an error + cleanup, err = operations.RemoveExisting(ctx, r.Fremote, file2.Path, "TEST") + assert.Equal(t, err, nil) + + // Check file2 with truncated temporary name exists + buf.Reset() + err = operations.List(ctx, r.Fremote, &buf) + require.NoError(t, err) + res = buf.String() + assert.NotContains(t, res, " 21 "+file2.Path+"\n") + assert.NotContains(t, res, " 21 "+file2.Path+".") + assert.Contains(t, res, " 21 "+file2.Path[:100]) + + returnedError = errors.New("BOOM") + cleanup(&returnedError) + r.CheckRemoteItems(t, file2) + + // Remove file2 + cleanup, err = operations.RemoveExisting(ctx, r.Fremote, file2.Path, "TEST") + assert.Equal(t, err, nil) + + // Check file2 with truncated temporary name exists + buf.Reset() + err = operations.List(ctx, r.Fremote, &buf) + require.NoError(t, err) + res = buf.String() + assert.NotContains(t, res, " 21 "+file2.Path+"\n") + assert.NotContains(t, res, " 21 "+file2.Path+".") + assert.Contains(t, res, " 21 "+file2.Path[:100]) + + returnedError = nil + cleanup(&returnedError) + r.CheckRemoteItems(t) +} diff --git a/fs/operations/operationsflags/operationsflags.go b/fs/operations/operationsflags/operationsflags.go new file mode 100644 index 0000000..054ed96 --- /dev/null +++ b/fs/operations/operationsflags/operationsflags.go @@ -0,0 +1,147 @@ +// Package operationsflags defines the flags used by rclone operations. +// It is decoupled into a separate package so it can be replaced. +package operationsflags + +import ( + "context" + _ "embed" + "io" + "os" + "strings" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/flags" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/operations" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +//go:embed operationsflags.md +var help string + +// Help returns the help string cleaned up to simplify appending +func Help() string { + return strings.TrimSpace(help) +} + +// AddLoggerFlagsOptions contains options for the Logger Flags +type AddLoggerFlagsOptions struct { + Combined string // a file with file names with leading sigils + MissingOnSrc string // files only in the destination + MissingOnDst string // files only in the source + Match string // matching files + Differ string // differing files + ErrFile string // files with errors of some kind + DestAfter string // files that exist on the destination post-sync +} + +// AnySet checks if any of the logger flags have a non-blank value +func (o AddLoggerFlagsOptions) AnySet() bool { + return anyNotBlank(o.Combined, o.MissingOnSrc, o.MissingOnDst, o.Match, o.Differ, o.ErrFile, o.DestAfter) +} + +func anyNotBlank(s ...string) bool { + for _, x := range s { + if x != "" { + return true + } + } + return false +} + +// AddLoggerFlags adds the logger flags to the cmdFlags command +func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt *AddLoggerFlagsOptions) { + flags.StringVarP(cmdFlags, &flagsOpt.Combined, "combined", "", flagsOpt.Combined, "Make a combined report of changes to this file", "Sync") + flags.StringVarP(cmdFlags, &flagsOpt.MissingOnSrc, "missing-on-src", "", flagsOpt.MissingOnSrc, "Report all files missing from the source to this file", "Sync") + flags.StringVarP(cmdFlags, &flagsOpt.MissingOnDst, "missing-on-dst", "", flagsOpt.MissingOnDst, "Report all files missing from the destination to this file", "Sync") + flags.StringVarP(cmdFlags, &flagsOpt.Match, "match", "", flagsOpt.Match, "Report all matching files to this file", "Sync") + flags.StringVarP(cmdFlags, &flagsOpt.Differ, "differ", "", flagsOpt.Differ, "Report all non-matching files to this file", "Sync") + flags.StringVarP(cmdFlags, &flagsOpt.ErrFile, "error", "", flagsOpt.ErrFile, "Report all files with errors (hashing or reading) to this file", "Sync") + flags.StringVarP(cmdFlags, &flagsOpt.DestAfter, "dest-after", "", flagsOpt.DestAfter, "Report all files that exist on the dest post-sync", "Sync") + + // lsf flags for destAfter + flags.StringVarP(cmdFlags, &opt.Format, "format", "F", "p", "Output format - see lsf help for details", "Sync") + flags.StringVarP(cmdFlags, &opt.TimeFormat, "timeformat", "t", "", "Specify a custom time format - see docs for details (default: 2006-01-02 15:04:05)", "") + flags.StringVarP(cmdFlags, &opt.Separator, "separator", "s", ";", "Separator for the items in the format", "Sync") + flags.BoolVarP(cmdFlags, &opt.DirSlash, "dir-slash", "d", true, "Append a slash to directory names", "Sync") + opt.HashType = hash.MD5 + flags.FVarP(cmdFlags, &opt.HashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash", "Sync") + flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", true, "Only list files", "Sync") + flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Only list directories", "Sync") + flags.BoolVarP(cmdFlags, &opt.Csv, "csv", "", false, "Output in CSV format", "Sync") + flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync") + // flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "") +} + +// ConfigureLoggers verifies and sets up writers for log files requested via CLI flags +func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, opt *operations.LoggerOpt, flagsOpt AddLoggerFlagsOptions) (func(), error) { + closers := []io.Closer{} + + if opt.TimeFormat == "max" { + opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision()) + } + opt.SetListFormat(ctx, command.Flags()) + opt.NewListJSON(ctx, fdst, "") + + open := func(name string, pout *io.Writer) error { + if name == "" { + return nil + } + if name == "-" { + *pout = os.Stdout + return nil + } + out, err := os.Create(name) + if err != nil { + return err + } + *pout = out + closers = append(closers, out) + return nil + } + + if err := open(flagsOpt.Combined, &opt.Combined); err != nil { + return nil, err + } + if err := open(flagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil { + return nil, err + } + if err := open(flagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil { + return nil, err + } + if err := open(flagsOpt.Match, &opt.Match); err != nil { + return nil, err + } + if err := open(flagsOpt.Differ, &opt.Differ); err != nil { + return nil, err + } + if err := open(flagsOpt.ErrFile, &opt.Error); err != nil { + return nil, err + } + if err := open(flagsOpt.DestAfter, &opt.DestAfter); err != nil { + return nil, err + } + + close := func() { + for _, closer := range closers { + err := closer.Close() + if err != nil { + fs.Errorf(nil, "Failed to close report output: %v", err) + } + } + } + + ci := fs.GetConfig(ctx) + if ci.NoTraverse && opt.Combined != nil { + fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n") + } + if ci.NoTraverse && opt.MissingOnSrc != nil { + fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n") + } + if ci.NoTraverse && opt.DestAfter != nil { + fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n") + } + + return close, nil +} diff --git a/fs/operations/operationsflags/operationsflags.md b/fs/operations/operationsflags/operationsflags.md new file mode 100644 index 0000000..43def48 --- /dev/null +++ b/fs/operations/operationsflags/operationsflags.md @@ -0,0 +1,40 @@ +### Logger Flags + +The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match` and `--error` +flags write paths, one per line, to the file name (or stdout if it is `-`) +supplied. What they write is described in the help below. For example +`--differ` will write all paths which are present on both the source and +destination but different. + +The `--combined` flag will write a file (or stdout) which contains all +file paths with a symbol and then a space and then the path to tell +you what happened to it. These are reminiscent of diff files. + +- `= path` means path was found in source and destination and was identical +- `- path` means path was missing on the source, so only in the destination +- `+ path` means path was missing on the destination, so only in the source +- `* path` means path was present in source and destination but different. +- `! path` means there was an error reading or hashing the source or dest. + +The `--dest-after` flag writes a list file using the same format flags +as [`lsf`](/commands/rclone_lsf/#synopsis) (including [customizable options +for hash, modtime, etc.](/commands/rclone_lsf/#synopsis)) +Conceptually it is similar to rsync's `--itemize-changes`, but not identical +-- it should output an accurate list of what will be on the destination +after the command is finished. + +When the `--no-traverse` flag is set, all logs involving files that exist only +on the destination will be incomplete or completely missing. + +Note that these logger flags have a few limitations, and certain scenarios +are not currently supported: + +- `--max-duration` / `CutoffModeHard` +- `--compare-dest` / `--copy-dest` +- server-side moves of an entire dir at once +- High-level retries, because there would be duplicates (use `--retries 1` to disable) +- Possibly some unusual error scenarios + +Note also that each file is logged during execution, as opposed to after, so it +is most useful as a predictor of what SHOULD happen to each file +(which may or may not match what actually DID). diff --git a/fs/operations/rc.go b/fs/operations/rc.go new file mode 100644 index 0000000..ce92a00 --- /dev/null +++ b/fs/operations/rc.go @@ -0,0 +1,1016 @@ +package operations + +import ( + "context" + "errors" + "fmt" + "io" + "mime" + "mime/multipart" + "net/http" + "path" + "strings" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/rc" + "github.com/rclone/rclone/lib/diskusage" +) + +func init() { + rc.Add(rc.Call{ + Path: "operations/list", + AuthRequired: true, + Fn: rcList, + Title: "List the given remote and path in JSON format", + Help: `This takes the following parameters: + +- fs - a remote name string e.g. "drive:" +- remote - a path within that remote e.g. "dir" +- opt - a dictionary of options to control the listing (optional) + - recurse - If set recurse directories + - noModTime - If set return modification time + - showEncrypted - If set show decrypted names + - showOrigIDs - If set show the IDs for each item if known + - showHash - If set return a dictionary of hashes + - noMimeType - If set don't show mime types + - dirsOnly - If set only show directories + - filesOnly - If set only show files + - metadata - If set return metadata of objects also + - hashTypes - array of strings of hash types to show if showHash set + +Returns: + +- list + - This is an array of objects as described in the lsjson command + +See the [lsjson](/commands/rclone_lsjson/) command for more information on the above and examples. +`, + }) +} + +// List the directory +func rcList(ctx context.Context, in rc.Params) (out rc.Params, err error) { + f, remote, err := rc.GetFsAndRemote(ctx, in) + if err != nil { + return nil, err + } + var opt ListJSONOpt + err = in.GetStruct("opt", &opt) + if rc.NotErrParamNotFound(err) { + return nil, err + } + list := []*ListJSONItem{} + err = ListJSON(ctx, f, remote, &opt, func(item *ListJSONItem) error { + list = append(list, item) + return nil + }) + if err != nil { + return nil, err + } + out = make(rc.Params) + out["list"] = list + return out, nil +} + +func init() { + rc.Add(rc.Call{ + Path: "operations/stat", + AuthRequired: true, + Fn: rcStat, + Title: "Give information about the supplied file or directory", + Help: `This takes the following parameters + +- fs - a remote name string eg "drive:" +- remote - a path within that remote eg "dir" +- opt - a dictionary of options to control the listing (optional) + - see operations/list for the options + +The result is + +- item - an object as described in the lsjson command. Will be null if not found. + +Note that if you are only interested in files then it is much more +efficient to set the filesOnly flag in the options. + +See the [lsjson](/commands/rclone_lsjson/) command for more information on the above and examples. +`, + }) +} + +// List the directory +func rcStat(ctx context.Context, in rc.Params) (out rc.Params, err error) { + f, remote, err := rc.GetFsAndRemote(ctx, in) + if err != nil { + return nil, err + } + var opt ListJSONOpt + err = in.GetStruct("opt", &opt) + if rc.NotErrParamNotFound(err) { + return nil, err + } + item, err := StatJSON(ctx, f, remote, &opt) + if err != nil { + return nil, err + } + out = make(rc.Params) + out["item"] = item + return out, nil +} + +func init() { + rc.Add(rc.Call{ + Path: "operations/about", + AuthRequired: true, + Fn: rcAbout, + Title: "Return the space used on the remote", + Help: `This takes the following parameters: + +- fs - a remote name string e.g. "drive:" + +The result is as returned from rclone about --json + +See the [about](/commands/rclone_about/) command for more information on the above. +`, + }) +} + +// About the remote +func rcAbout(ctx context.Context, in rc.Params) (out rc.Params, err error) { + f, err := rc.GetFs(ctx, in) + if err != nil { + return nil, err + } + doAbout := f.Features().About + if doAbout == nil { + return nil, fmt.Errorf("%v doesn't support about", f) + } + u, err := doAbout(ctx) + if err != nil { + return nil, fmt.Errorf("about call failed: %w", err) + } + err = rc.Reshape(&out, u) + if err != nil { + return nil, fmt.Errorf("about Reshape failed: %w", err) + } + return out, nil +} + +func init() { + for _, copy := range []bool{false, true} { + name := "Move" + if copy { + name = "Copy" + } + rc.Add(rc.Call{ + Path: "operations/" + strings.ToLower(name) + "file", + AuthRequired: true, + Fn: func(ctx context.Context, in rc.Params) (rc.Params, error) { + return rcMoveOrCopyFile(ctx, in, copy) + }, + Title: name + " a file from source remote to destination remote", + Help: `This takes the following parameters: + +- srcFs - a remote name string e.g. "drive:" for the source, "/" for local filesystem +- srcRemote - a path within that remote e.g. "file.txt" for the source +- dstFs - a remote name string e.g. "drive2:" for the destination, "/" for local filesystem +- dstRemote - a path within that remote e.g. "file2.txt" for the destination +`, + }) + } +} + +// Copy a file +func rcMoveOrCopyFile(ctx context.Context, in rc.Params, cp bool) (out rc.Params, err error) { + srcFs, srcRemote, err := rc.GetFsAndRemoteNamed(ctx, in, "srcFs", "srcRemote") + if err != nil { + return nil, err + } + dstFs, dstRemote, err := rc.GetFsAndRemoteNamed(ctx, in, "dstFs", "dstRemote") + if err != nil { + return nil, err + } + return nil, moveOrCopyFile(ctx, dstFs, srcFs, dstRemote, srcRemote, cp, false) +} + +func init() { + for _, op := range []struct { + name string + title string + help string + noRemote bool + needsRequest bool + noCommand bool + }{ + {name: "mkdir", title: "Make a destination directory or container"}, + {name: "rmdir", title: "Remove an empty directory or container"}, + {name: "purge", title: "Remove a directory or container and all of its contents"}, + {name: "rmdirs", title: "Remove all the empty directories in the path", help: "- leaveRoot - boolean, set to true not to delete the root\n"}, + {name: "delete", title: "Remove files in the path", noRemote: true}, + {name: "deletefile", title: "Remove the single file pointed to"}, + {name: "copyurl", title: "Copy the URL to the object", help: "- url - string, URL to read from\n - autoFilename - boolean, set to true to retrieve destination file name from url\n"}, + {name: "uploadfile", title: "Upload file using multiform/form-data", help: "- each part in body represents a file to be uploaded\n", needsRequest: true, noCommand: true}, + {name: "cleanup", title: "Remove trashed files in the remote or path", noRemote: true}, + {name: "settier", title: "Changes storage tier or class on all files in the path", noRemote: true}, + {name: "settierfile", title: "Changes storage tier or class on the single file pointed to", noCommand: true}, + } { + var remote, command string + if !op.noRemote { + remote = "- remote - a path within that remote e.g. \"dir\"\n" + } + if !op.noCommand { + command = "See the [" + op.name + "](/commands/rclone_" + op.name + "/) command for more information on the above.\n" + } + rc.Add(rc.Call{ + Path: "operations/" + op.name, + AuthRequired: true, + NeedsRequest: op.needsRequest, + Fn: func(ctx context.Context, in rc.Params) (rc.Params, error) { + return rcSingleCommand(ctx, in, op.name, op.noRemote) + }, + Title: op.title, + Help: `This takes the following parameters: + +- fs - a remote name string e.g. "drive:" +` + remote + op.help + "\n" + command, + }) + } +} + +// Run a single command, e.g. Mkdir +func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bool) (out rc.Params, err error) { + var ( + f fs.Fs + remote string + ) + if noRemote { + f, err = rc.GetFs(ctx, in) + } else { + f, remote, err = rc.GetFsAndRemote(ctx, in) + } + if err != nil { + return nil, err + } + switch name { + case "mkdir": + return nil, Mkdir(ctx, f, remote) + case "rmdir": + return nil, Rmdir(ctx, f, remote) + case "purge": + return nil, Purge(ctx, f, remote) + case "rmdirs": + leaveRoot, err := in.GetBool("leaveRoot") + if rc.NotErrParamNotFound(err) { + return nil, err + } + return nil, Rmdirs(ctx, f, remote, leaveRoot) + case "delete": + return nil, Delete(ctx, f) + case "deletefile": + o, err := f.NewObject(ctx, remote) + if err != nil { + return nil, err + } + return nil, DeleteFile(ctx, o) + case "copyurl": + url, err := in.GetString("url") + if err != nil { + return nil, err + } + autoFilename, _ := in.GetBool("autoFilename") + noClobber, _ := in.GetBool("noClobber") + headerFilename, _ := in.GetBool("headerFilename") + + _, err = CopyURL(ctx, f, remote, url, autoFilename, headerFilename, noClobber) + return nil, err + case "uploadfile": + + var request *http.Request + request, err := in.GetHTTPRequest() + if err != nil { + return nil, err + } + + contentType := request.Header.Get("Content-Type") + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil { + return nil, err + } + + if strings.HasPrefix(mediaType, "multipart/") { + mr := multipart.NewReader(request.Body, params["boundary"]) + for { + p, err := mr.NextPart() + if err == io.EOF { + return nil, nil + } + if err != nil { + return nil, err + } + if p.FileName() != "" { + obj, err := Rcat(ctx, f, path.Join(remote, p.FileName()), p, time.Now(), nil) + if err != nil { + return nil, err + } + fs.Debugf(obj, "Upload Succeeded") + } + } + } + return nil, nil + case "cleanup": + return nil, CleanUp(ctx, f) + case "settier": + if !f.Features().SetTier { + return nil, fmt.Errorf("remote %s does not support settier", f.Name()) + } + tier, err := in.GetString("tier") + if err != nil { + return nil, err + } + return nil, SetTier(ctx, f, tier) + case "settierfile": + if !f.Features().SetTier { + return nil, fmt.Errorf("remote %s does not support settier", f.Name()) + } + tier, err := in.GetString("tier") + if err != nil { + return nil, err + } + o, err := f.NewObject(ctx, remote) + if err != nil { + return nil, err + } + return nil, SetTierFile(ctx, o, tier) + } + panic("unknown rcSingleCommand type") +} + +func init() { + rc.Add(rc.Call{ + Path: "operations/size", + AuthRequired: true, + Fn: rcSize, + Title: "Count the number of bytes and files in remote", + Help: `This takes the following parameters: + +- fs - a remote name string e.g. "drive:path/to/dir" + +Returns: + +- count - number of files +- bytes - number of bytes in those files + +See the [size](/commands/rclone_size/) command for more information on the above. +`, + }) +} + +// Size a directory +func rcSize(ctx context.Context, in rc.Params) (out rc.Params, err error) { + f, err := rc.GetFs(ctx, in) + if err != nil { + return nil, err + } + count, bytes, sizeless, err := Count(ctx, f) + if err != nil { + return nil, err + } + out = make(rc.Params) + out["count"] = count + out["bytes"] = bytes + out["sizeless"] = sizeless + return out, nil +} + +func init() { + rc.Add(rc.Call{ + Path: "operations/publiclink", + AuthRequired: true, + Fn: rcPublicLink, + Title: "Create or retrieve a public link to the given file or folder.", + Help: `This takes the following parameters: + +- fs - a remote name string e.g. "drive:" +- remote - a path within that remote e.g. "dir" +- unlink - boolean - if set removes the link rather than adding it (optional) +- expire - string - the expiry time of the link e.g. "1d" (optional) + +Returns: + +- url - URL of the resource + +See the [link](/commands/rclone_link/) command for more information on the above. +`, + }) +} + +// Make a public link +func rcPublicLink(ctx context.Context, in rc.Params) (out rc.Params, err error) { + f, remote, err := rc.GetFsAndRemote(ctx, in) + if err != nil { + return nil, err + } + unlink, _ := in.GetBool("unlink") + expire, err := in.GetDuration("expire") + if rc.IsErrParamNotFound(err) { + expire = time.Duration(fs.DurationOff) + } else if err != nil { + return nil, err + } + url, err := PublicLink(ctx, f, remote, fs.Duration(expire), unlink) + if err != nil { + return nil, err + } + out = make(rc.Params) + out["url"] = url + return out, nil +} + +func init() { + rc.Add(rc.Call{ + Path: "operations/fsinfo", + Fn: rcFsInfo, + Title: "Return information about the remote", + Help: `This takes the following parameters: + +- fs - a remote name string e.g. "drive:" + +This returns info about the remote passed in; + +` + "```" + ` +{ + // optional features and whether they are available or not + "Features": { + "About": true, + "BucketBased": false, + "BucketBasedRootOK": false, + "CanHaveEmptyDirectories": true, + "CaseInsensitive": false, + "ChangeNotify": false, + "CleanUp": false, + "Command": true, + "Copy": false, + "DirCacheFlush": false, + "DirMove": true, + "Disconnect": false, + "DuplicateFiles": false, + "GetTier": false, + "IsLocal": true, + "ListR": false, + "MergeDirs": false, + "MetadataInfo": true, + "Move": true, + "OpenWriterAt": true, + "PublicLink": false, + "Purge": true, + "PutStream": true, + "PutUnchecked": false, + "ReadMetadata": true, + "ReadMimeType": false, + "ServerSideAcrossConfigs": false, + "SetTier": false, + "SetWrapper": false, + "Shutdown": false, + "SlowHash": true, + "SlowModTime": false, + "UnWrap": false, + "UserInfo": false, + "UserMetadata": true, + "WrapFs": false, + "WriteMetadata": true, + "WriteMimeType": false + }, + // Names of hashes available + "Hashes": [ + "md5", + "sha1", + "whirlpool", + "crc32", + "sha256", + "dropbox", + "mailru", + "quickxor" + ], + "Name": "local", // Name as created + "Precision": 1, // Precision of timestamps in ns + "Root": "/", // Path as created + "String": "Local file system at /", // how the remote will appear in logs + // Information about the system metadata for this backend + "MetadataInfo": { + "System": { + "atime": { + "Help": "Time of last access", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "btime": { + "Help": "Time of file birth (creation)", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "gid": { + "Help": "Group ID of owner", + "Type": "decimal number", + "Example": "500" + }, + "mode": { + "Help": "File type and mode", + "Type": "octal, unix style", + "Example": "0100664" + }, + "mtime": { + "Help": "Time of last modification", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "rdev": { + "Help": "Device ID (if special file)", + "Type": "hexadecimal", + "Example": "1abc" + }, + "uid": { + "Help": "User ID of owner", + "Type": "decimal number", + "Example": "500" + } + }, + "Help": "Textual help string\n" + } +} +` + "```" + ` + +This command does not have a command line equivalent so use this instead: + + rclone rc --loopback operations/fsinfo fs=remote: + +`, + }) +} + +// Fsinfo the remote +func rcFsInfo(ctx context.Context, in rc.Params) (out rc.Params, err error) { + f, err := rc.GetFs(ctx, in) + if err != nil { + return nil, err + } + info := GetFsInfo(f) + err = rc.Reshape(&out, info) + if err != nil { + return nil, fmt.Errorf("fsinfo Reshape failed: %w", err) + } + return out, nil +} + +func init() { + rc.Add(rc.Call{ + Path: "backend/command", + AuthRequired: true, + Fn: rcBackend, + Title: "Runs a backend command.", + Help: `This takes the following parameters: + +- command - a string with the command name +- fs - a remote name string e.g. "drive:" +- arg - a list of arguments for the backend command +- opt - a map of string to string of options + +Returns: + +- result - result from the backend command + +Example: + + rclone rc backend/command command=noop fs=. -o echo=yes -o blue -a path1 -a path2 + +Returns + +` + "```" + ` +{ + "result": { + "arg": [ + "path1", + "path2" + ], + "name": "noop", + "opt": { + "blue": "", + "echo": "yes" + } + } +} +` + "```" + ` + +Note that this is the direct equivalent of using this "backend" +command: + + rclone backend noop . -o echo=yes -o blue path1 path2 + +Note that arguments must be preceded by the "-a" flag + +See the [backend](/commands/rclone_backend/) command for more information. +`, + }) +} + +// Make a public link +func rcBackend(ctx context.Context, in rc.Params) (out rc.Params, err error) { + f, err := rc.GetFs(ctx, in) + if err != nil { + return nil, err + } + doCommand := f.Features().Command + if doCommand == nil { + return nil, fmt.Errorf("%v: doesn't support backend commands", f) + } + command, err := in.GetString("command") + if err != nil { + return nil, err + } + opt := map[string]string{} + err = in.GetStructMissingOK("opt", &opt) + if err != nil { + return nil, err + } + arg := []string{} + err = in.GetStructMissingOK("arg", &arg) + if err != nil { + return nil, err + } + result, err := doCommand(ctx, command, arg, opt) + if err != nil { + return nil, fmt.Errorf("command %q failed: %w", command, err) + } + out = make(rc.Params) + out["result"] = result + return out, nil +} + +// This should really be in fs/rc/internal.go but can't go there due +// to a circular dependency on config. +func init() { + rc.Add(rc.Call{ + Path: "core/du", + Fn: rcDu, + Title: "Returns disk usage of a locally attached disk.", + Help: ` +This returns the disk usage for the local directory passed in as dir. + +If the directory is not passed in, it defaults to the directory +pointed to by --cache-dir. + +- dir - string (optional) + +Returns: + +` + "```" + ` +{ + "dir": "/", + "info": { + "Available": 361769115648, + "Free": 361785892864, + "Total": 982141468672 + } +} +` + "```" + ` +`, + }) +} + +// Terminates app +func rcDu(ctx context.Context, in rc.Params) (out rc.Params, err error) { + dir, err := in.GetString("dir") + if rc.IsErrParamNotFound(err) { + dir = config.GetCacheDir() + } else if err != nil { + return nil, err + } + info, err := diskusage.New(dir) + if err != nil { + return nil, err + } + out = rc.Params{ + "dir": dir, + "info": info, + } + return out, nil +} + +func init() { + rc.Add(rc.Call{ + Path: "operations/check", + AuthRequired: true, + Fn: rcCheck, + Title: "check the source and destination are the same", + Help: `Checks the files in the source and destination match. It compares +sizes and hashes and logs a report of files that don't +match. It doesn't alter the source or destination. + +This takes the following parameters: + +- srcFs - a remote name string e.g. "drive:" for the source, "/" for local filesystem +- dstFs - a remote name string e.g. "drive2:" for the destination, "/" for local filesystem +- download - check by downloading rather than with hash +- checkFileHash - treat checkFileFs:checkFileRemote as a SUM file with hashes of given type +- checkFileFs - treat checkFileFs:checkFileRemote as a SUM file with hashes of given type +- checkFileRemote - treat checkFileFs:checkFileRemote as a SUM file with hashes of given type +- oneWay - check one way only, source files must exist on remote +- combined - make a combined report of changes (default false) +- missingOnSrc - report all files missing from the source (default true) +- missingOnDst - report all files missing from the destination (default true) +- match - report all matching files (default false) +- differ - report all non-matching files (default true) +- error - report all files with errors (hashing or reading) (default true) + +If you supply the download flag, it will download the data from +both remotes and check them against each other on the fly. This can +be useful for remotes that don't support hashes or if you really want +to check all the data. + +If you supply the size-only global flag, it will only compare the sizes not +the hashes as well. Use this for a quick check. + +If you supply the checkFileHash option with a valid hash name, the +checkFileFs:checkFileRemote must point to a text file in the SUM +format. This treats the checksum file as the source and dstFs as the +destination. Note that srcFs is not used and should not be supplied in +this case. + +Returns: + +- success - true if no error, false otherwise +- status - textual summary of check, OK or text string +- hashType - hash used in check, may be missing +- combined - array of strings of combined report of changes +- missingOnSrc - array of strings of all files missing from the source +- missingOnDst - array of strings of all files missing from the destination +- match - array of strings of all matching files +- differ - array of strings of all non-matching files +- error - array of strings of all files with errors (hashing or reading) + +`, + }) +} + +// Writer which writes into the slice provided +type stringWriter struct { + out *[]string +} + +// Write writes len(p) bytes from p to the underlying data stream. It returns +// the number of bytes written from p (0 <= n <= len(p)) and any error +// encountered that caused the write to stop early. Write must return a non-nil +// error if it returns n < len(p). Write must not modify the slice data, +// even temporarily. +// +// Implementations must not retain p. +func (s stringWriter) Write(p []byte) (n int, err error) { + result := string(p) + result = strings.TrimSuffix(result, "\n") + *s.out = append(*s.out, result) + return len(p), nil +} + +// Check two directories +func rcCheck(ctx context.Context, in rc.Params) (out rc.Params, err error) { + srcFs, err := rc.GetFsNamed(ctx, in, "srcFs") + if err != nil && !rc.IsErrParamNotFound(err) { + return nil, err + } + + dstFs, err := rc.GetFsNamed(ctx, in, "dstFs") + if err != nil { + return nil, err + } + + checkFileFs, checkFileRemote, err := rc.GetFsAndRemoteNamed(ctx, in, "checkFileFs", "checkFileRemote") + if err != nil && !rc.IsErrParamNotFound(err) { + return nil, err + } + + checkFileHash, err := in.GetString("checkFileHash") + if err != nil && !rc.IsErrParamNotFound(err) { + return nil, err + } + + checkFileSet := 0 + if checkFileHash != "" { + checkFileSet++ + } + if checkFileFs != nil { + checkFileSet++ + } + if checkFileRemote != "" { + checkFileSet++ + } + if checkFileSet > 0 && checkFileSet < 3 { + return nil, fmt.Errorf("need all of checkFileFs, checkFileRemote, checkFileHash to be set together") + } + + var checkFileHashType hash.Type + if checkFileHash != "" { + if err := checkFileHashType.Set(checkFileHash); err != nil { + return nil, err + } + if srcFs != nil { + return nil, rc.NewErrParamInvalid(errors.New("only supply dstFs when using checkFileHash")) + } + } else if srcFs == nil { + return nil, rc.NewErrParamInvalid(errors.New("need srcFs parameter when not using checkFileHash")) + } + + oneway, _ := in.GetBool("oneWay") + download, _ := in.GetBool("download") + + opt := &CheckOpt{ + Fsrc: srcFs, + Fdst: dstFs, + OneWay: oneway, + } + + out = rc.Params{} + + getOutput := func(name string, Default bool) io.Writer { + active, err := in.GetBool(name) + if err != nil { + active = Default + } + if !active { + return nil + } + result := []string{} + out[name] = &result + return stringWriter{&result} + } + + opt.Combined = getOutput("combined", false) + opt.MissingOnSrc = getOutput("missingOnSrc", true) + opt.MissingOnDst = getOutput("missingOnDst", true) + opt.Match = getOutput("match", false) + opt.Differ = getOutput("differ", true) + opt.Error = getOutput("error", true) + + if checkFileHash != "" { + out["hashType"] = checkFileHashType.String() + err = CheckSum(ctx, dstFs, checkFileFs, checkFileRemote, checkFileHashType, opt, download) + } else { + if download { + err = CheckDownload(ctx, opt) + } else { + out["hashType"] = srcFs.Hashes().Overlap(dstFs.Hashes()).GetOne().String() + err = Check(ctx, opt) + } + } + if err != nil { + out["status"] = err.Error() + out["success"] = false + } else { + out["status"] = "OK" + out["success"] = true + } + return out, nil +} + +func init() { + rc.Add(rc.Call{ + Path: "operations/hashsum", + AuthRequired: true, + Fn: rcHashsum, + Title: "Produces a hashsum file for all the objects in the path.", + Help: `Produces a hash file for all the objects in the path using the hash +named. The output is in the same format as the standard +md5sum/sha1sum tool. + +This takes the following parameters: + +- fs - a remote name string e.g. "drive:" for the source, "/" for local filesystem + - this can point to a file and just that file will be returned in the listing. +- hashType - type of hash to be used +- download - check by downloading rather than with hash (boolean) +- base64 - output the hashes in base64 rather than hex (boolean) + +If you supply the download flag, it will download the data from the +remote and create the hash on the fly. This can be useful for remotes +that don't support the given hash or if you really want to check all +the data. + +Note that if you wish to supply a checkfile to check hashes against +the current files then you should use operations/check instead of +operations/hashsum. + +Returns: + +- hashsum - array of strings of the hashes +- hashType - type of hash used + +Example: + + $ rclone rc --loopback operations/hashsum fs=bin hashType=MD5 download=true base64=true + { + "hashType": "md5", + "hashsum": [ + "WTSVLpuiXyJO_kGzJerRLg== backend-versions.sh", + "v1b_OlWCJO9LtNq3EIKkNQ== bisect-go-rclone.sh", + "VHbmHzHh4taXzgag8BAIKQ== bisect-rclone.sh", + ] + } + +See the [hashsum](/commands/rclone_hashsum/) command for more information on the above. +`, + }) +} + +// Parse download, base64 and hashType parameters +func parseHashParameters(in rc.Params) (download bool, base64 bool, ht hash.Type, err error) { + download, _ = in.GetBool("download") + base64, _ = in.GetBool("base64") + hashType, err := in.GetString("hashType") + if err != nil { + return + } + err = ht.Set(hashType) + return +} + +// Hashsum a directory +func rcHashsum(ctx context.Context, in rc.Params) (out rc.Params, err error) { + ctx, f, err := rc.GetFsNamedFileOK(ctx, in, "fs") + if err != nil { + return nil, err + } + + download, base64, ht, err := parseHashParameters(in) + if err != nil { + return out, err + } + + hashes := []string{} + err = HashLister(ctx, ht, base64, download, f, stringWriter{&hashes}) + out = rc.Params{ + "hashType": ht.String(), + "hashsum": hashes, + } + return out, err +} + +func init() { + rc.Add(rc.Call{ + Path: "operations/hashsumfile", + AuthRequired: true, + Fn: rcHashsumFile, + Title: "Produces a hash for a single file.", + Help: `Produces a hash for a single file using the hash named. + +This takes the following parameters: + +- fs - a remote name string e.g. "drive:" +- remote - a path within that remote e.g. "file.txt" +- hashType - type of hash to be used +- download - check by downloading rather than with hash (boolean) +- base64 - output the hashes in base64 rather than hex (boolean) + +If you supply the download flag, it will download the data from the +remote and create the hash on the fly. This can be useful for remotes +that don't support the given hash or if you really want to read all +the data. + +Returns: + +- hash - hash for the file +- hashType - type of hash used + +Example: + + $ rclone rc --loopback operations/hashsumfile fs=/ remote=/bin/bash hashType=MD5 download=true base64=true + { + "hashType": "md5", + "hash": "MDMw-fG2YXs7Uz5Nz-H68A==" + } + +See the [hashsum](/commands/rclone_hashsum/) command for more information on the above. +`, + }) +} + +// Hashsum a file +func rcHashsumFile(ctx context.Context, in rc.Params) (out rc.Params, err error) { + f, remote, err := rc.GetFsAndRemote(ctx, in) + if err != nil { + return nil, err + } + download, base64, ht, err := parseHashParameters(in) + if err != nil { + return out, err + } + o, err := f.NewObject(ctx, remote) + if err != nil { + return nil, err + } + sum, err := HashSum(ctx, ht, base64, download, o) + out = rc.Params{ + "hashType": ht.String(), + "hash": sum, + } + return out, err +} diff --git a/fs/operations/rc_test.go b/fs/operations/rc_test.go new file mode 100644 index 0000000..04357fb --- /dev/null +++ b/fs/operations/rc_test.go @@ -0,0 +1,892 @@ +package operations_test + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path" + "sort" + "strings" + "testing" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/rc" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/lib/diskusage" + "github.com/rclone/rclone/lib/rest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func rcNewRun(t *testing.T, method string) (*fstest.Run, *rc.Call) { + if *fstest.RemoteName != "" { + t.Skip("Skipping test on non local remote") + } + r := fstest.NewRun(t) + call := rc.Calls.Get(method) + assert.NotNil(t, call) + cache.Put(r.LocalName, r.Flocal) + cache.Put(r.FremoteName, r.Fremote) + return r, call +} + +// operations/about: Return the space used on the remote +func TestRcAbout(t *testing.T) { + r, call := rcNewRun(t, "operations/about") + r.Mkdir(context.Background(), r.Fremote) + + // Will get an error if remote doesn't support About + expectedErr := r.Fremote.Features().About == nil + + in := rc.Params{ + "fs": r.FremoteName, + } + out, err := call.Fn(context.Background(), in) + if expectedErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + + // Can't really check the output much! + assert.NotEqual(t, int64(0), out["Total"]) +} + +// operations/cleanup: Remove trashed files in the remote or path +func TestRcCleanup(t *testing.T) { + r, call := rcNewRun(t, "operations/cleanup") + + in := rc.Params{ + "fs": r.LocalName, + } + out, err := call.Fn(context.Background(), in) + require.Error(t, err) + assert.Equal(t, rc.Params(nil), out) + assert.Contains(t, err.Error(), "doesn't support cleanup") +} + +// operations/copyfile: Copy a file from source remote to destination remote +func TestRcCopyfile(t *testing.T) { + r, call := rcNewRun(t, "operations/copyfile") + file1 := r.WriteFile("file1", "file1 contents", t1) + r.Mkdir(context.Background(), r.Fremote) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t) + + in := rc.Params{ + "srcFs": r.LocalName, + "srcRemote": "file1", + "dstFs": r.FremoteName, + "dstRemote": "file1-renamed", + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + r.CheckLocalItems(t, file1) + file1.Path = "file1-renamed" + r.CheckRemoteItems(t, file1) +} + +// operations/copyurl: Copy the URL to the object +func TestRcCopyurl(t *testing.T) { + r, call := rcNewRun(t, "operations/copyurl") + contents := "file1 contents\n" + file1 := r.WriteFile("file1", contents, t1) + r.Mkdir(context.Background(), r.Fremote) + r.CheckRemoteItems(t) + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, err := w.Write([]byte(contents)) + assert.NoError(t, err) + })) + defer ts.Close() + + in := rc.Params{ + "fs": r.FremoteName, + "remote": "file1", + "url": ts.URL, + "autoFilename": false, + "noClobber": false, + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + in = rc.Params{ + "fs": r.FremoteName, + "remote": "file1", + "url": ts.URL, + "autoFilename": false, + "noClobber": true, + } + out, err = call.Fn(context.Background(), in) + require.Error(t, err) + assert.Equal(t, rc.Params(nil), out) + + urlFileName := "filename.txt" + in = rc.Params{ + "fs": r.FremoteName, + "remote": "", + "url": ts.URL + "/" + urlFileName, + "autoFilename": true, + "noClobber": false, + } + out, err = call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + in = rc.Params{ + "fs": r.FremoteName, + "remote": "", + "url": ts.URL, + "autoFilename": true, + "noClobber": false, + } + out, err = call.Fn(context.Background(), in) + require.Error(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported) +} + +// operations/delete: Remove files in the path +func TestRcDelete(t *testing.T) { + r, call := rcNewRun(t, "operations/delete") + + file1 := r.WriteObject(context.Background(), "small", "1234567890", t2) // 10 bytes + file2 := r.WriteObject(context.Background(), "medium", "------------------------------------------------------------", t1) // 60 bytes + file3 := r.WriteObject(context.Background(), "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes + r.CheckRemoteItems(t, file1, file2, file3) + + in := rc.Params{ + "fs": r.FremoteName, + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + r.CheckRemoteItems(t) +} + +// operations/deletefile: Remove the single file pointed to +func TestRcDeletefile(t *testing.T) { + r, call := rcNewRun(t, "operations/deletefile") + + file1 := r.WriteObject(context.Background(), "small", "1234567890", t2) // 10 bytes + file2 := r.WriteObject(context.Background(), "medium", "------------------------------------------------------------", t1) // 60 bytes + r.CheckRemoteItems(t, file1, file2) + + in := rc.Params{ + "fs": r.FremoteName, + "remote": "small", + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + r.CheckRemoteItems(t, file2) +} + +// operations/list: List the given remote and path in JSON format. +func TestRcList(t *testing.T) { + r, call := rcNewRun(t, "operations/list") + + file1 := r.WriteObject(context.Background(), "a", "a", t1) + file2 := r.WriteObject(context.Background(), "subdir/b", "bb", t2) + + r.CheckRemoteItems(t, file1, file2) + + in := rc.Params{ + "fs": r.FremoteName, + "remote": "", + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + + list := out["list"].([]*operations.ListJSONItem) + assert.Equal(t, 2, len(list)) + + checkFile1 := func(got *operations.ListJSONItem) { + assert.WithinDuration(t, t1, got.ModTime.When, time.Second) + assert.Equal(t, "a", got.Path) + assert.Equal(t, "a", got.Name) + assert.Equal(t, int64(1), got.Size) + assert.Equal(t, "application/octet-stream", got.MimeType) + assert.Equal(t, false, got.IsDir) + } + checkFile1(list[0]) + + checkSubdir := func(got *operations.ListJSONItem) { + assert.Equal(t, "subdir", got.Path) + assert.Equal(t, "subdir", got.Name) + // assert.Equal(t, int64(-1), got.Size) // size can vary for directories + assert.Equal(t, "inode/directory", got.MimeType) + assert.Equal(t, true, got.IsDir) + } + checkSubdir(list[1]) + + in = rc.Params{ + "fs": r.FremoteName, + "remote": "", + "opt": rc.Params{ + "recurse": true, + }, + } + out, err = call.Fn(context.Background(), in) + require.NoError(t, err) + + list = out["list"].([]*operations.ListJSONItem) + assert.Equal(t, 3, len(list)) + checkFile1(list[0]) + checkSubdir(list[1]) + + checkFile2 := func(got *operations.ListJSONItem) { + assert.WithinDuration(t, t2, got.ModTime.When, time.Second) + assert.Equal(t, "subdir/b", got.Path) + assert.Equal(t, "b", got.Name) + assert.Equal(t, int64(2), got.Size) + assert.Equal(t, "application/octet-stream", got.MimeType) + assert.Equal(t, false, got.IsDir) + } + checkFile2(list[2]) +} + +// operations/stat: Stat the given remote and path in JSON format. +func TestRcStat(t *testing.T) { + r, call := rcNewRun(t, "operations/stat") + + file1 := r.WriteObject(context.Background(), "subdir/a", "a", t1) + + r.CheckRemoteItems(t, file1) + + fetch := func(t *testing.T, remotePath string) *operations.ListJSONItem { + in := rc.Params{ + "fs": r.FremoteName, + "remote": remotePath, + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + return out["item"].(*operations.ListJSONItem) + } + + t.Run("Root", func(t *testing.T) { + stat := fetch(t, "") + assert.Equal(t, "", stat.Path) + assert.Equal(t, "", stat.Name) + assert.Equal(t, int64(-1), stat.Size) + assert.Equal(t, "inode/directory", stat.MimeType) + assert.Equal(t, true, stat.IsDir) + }) + + t.Run("File", func(t *testing.T) { + stat := fetch(t, "subdir/a") + assert.WithinDuration(t, t1, stat.ModTime.When, time.Second) + assert.Equal(t, "subdir/a", stat.Path) + assert.Equal(t, "a", stat.Name) + assert.Equal(t, int64(1), stat.Size) + assert.Equal(t, "application/octet-stream", stat.MimeType) + assert.Equal(t, false, stat.IsDir) + }) + + t.Run("Subdir", func(t *testing.T) { + stat := fetch(t, "subdir") + assert.Equal(t, "subdir", stat.Path) + assert.Equal(t, "subdir", stat.Name) + // assert.Equal(t, int64(-1), stat.Size) // size can vary for directories + assert.Equal(t, "inode/directory", stat.MimeType) + assert.Equal(t, true, stat.IsDir) + }) + + t.Run("NotFound", func(t *testing.T) { + stat := fetch(t, "notfound") + assert.Nil(t, stat) + }) +} + +// operations/settier: Set the storage tier of a fs +func TestRcSetTier(t *testing.T) { + ctx := context.Background() + r, call := rcNewRun(t, "operations/settier") + if !r.Fremote.Features().SetTier { + t.Skip("settier not supported") + } + file1 := r.WriteObject(context.Background(), "file1", "file1 contents", t1) + r.CheckRemoteItems(t, file1) + + // Because we don't know what the current tier options here are, let's + // just get the current tier, and reuse that + o, err := r.Fremote.NewObject(ctx, file1.Path) + require.NoError(t, err) + trr, ok := o.(fs.GetTierer) + require.True(t, ok) + ctier := trr.GetTier() + in := rc.Params{ + "fs": r.FremoteName, + "tier": ctier, + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + +} + +// operations/settier: Set the storage tier of a file +func TestRcSetTierFile(t *testing.T) { + ctx := context.Background() + r, call := rcNewRun(t, "operations/settierfile") + if !r.Fremote.Features().SetTier { + t.Skip("settier not supported") + } + file1 := r.WriteObject(context.Background(), "file1", "file1 contents", t1) + r.CheckRemoteItems(t, file1) + + // Because we don't know what the current tier options here are, let's + // just get the current tier, and reuse that + o, err := r.Fremote.NewObject(ctx, file1.Path) + require.NoError(t, err) + trr, ok := o.(fs.GetTierer) + require.True(t, ok) + ctier := trr.GetTier() + in := rc.Params{ + "fs": r.FremoteName, + "remote": "file1", + "tier": ctier, + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + +} + +// operations/mkdir: Make a destination directory or container +func TestRcMkdir(t *testing.T) { + ctx := context.Background() + r, call := rcNewRun(t, "operations/mkdir") + r.Mkdir(context.Background(), r.Fremote) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote)) + + in := rc.Params{ + "fs": r.FremoteName, + "remote": "subdir", + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(ctx, r.Fremote)) +} + +// operations/movefile: Move a file from source remote to destination remote +func TestRcMovefile(t *testing.T) { + r, call := rcNewRun(t, "operations/movefile") + file1 := r.WriteFile("file1", "file1 contents", t1) + r.Mkdir(context.Background(), r.Fremote) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t) + + in := rc.Params{ + "srcFs": r.LocalName, + "srcRemote": "file1", + "dstFs": r.FremoteName, + "dstRemote": "file1-renamed", + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + r.CheckLocalItems(t) + file1.Path = "file1-renamed" + r.CheckRemoteItems(t, file1) +} + +// operations/purge: Remove a directory or container and all of its contents +func TestRcPurge(t *testing.T) { + ctx := context.Background() + r, call := rcNewRun(t, "operations/purge") + file1 := r.WriteObject(context.Background(), "subdir/file1", "subdir/file1 contents", t1) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"subdir"}, fs.GetModifyWindow(ctx, r.Fremote)) + + in := rc.Params{ + "fs": r.FremoteName, + "remote": "subdir", + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote)) +} + +// operations/rmdir: Remove an empty directory or container +func TestRcRmdir(t *testing.T) { + ctx := context.Background() + r, call := rcNewRun(t, "operations/rmdir") + r.Mkdir(context.Background(), r.Fremote) + assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir")) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(ctx, r.Fremote)) + + in := rc.Params{ + "fs": r.FremoteName, + "remote": "subdir", + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote)) +} + +// operations/rmdirs: Remove all the empty directories in the path +func TestRcRmdirs(t *testing.T) { + ctx := context.Background() + r, call := rcNewRun(t, "operations/rmdirs") + r.Mkdir(context.Background(), r.Fremote) + assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir")) + assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir/subsubdir")) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir", "subdir/subsubdir"}, fs.GetModifyWindow(ctx, r.Fremote)) + + in := rc.Params{ + "fs": r.FremoteName, + "remote": "subdir", + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote)) + + assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir")) + assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir/subsubdir")) + + in = rc.Params{ + "fs": r.FremoteName, + "remote": "subdir", + "leaveRoot": true, + } + out, err = call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(ctx, r.Fremote)) + +} + +// operations/size: Count the number of bytes and files in remote +func TestRcSize(t *testing.T) { + r, call := rcNewRun(t, "operations/size") + file1 := r.WriteObject(context.Background(), "small", "1234567890", t2) // 10 bytes + file2 := r.WriteObject(context.Background(), "subdir/medium", "------------------------------------------------------------", t1) // 60 bytes + file3 := r.WriteObject(context.Background(), "subdir/subsubdir/large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 50 bytes + r.CheckRemoteItems(t, file1, file2, file3) + + in := rc.Params{ + "fs": r.FremoteName, + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params{ + "count": int64(3), + "bytes": int64(120), + "sizeless": int64(0), + }, out) +} + +// operations/publiclink: Create or retrieve a public link to the given file or folder. +func TestRcPublicLink(t *testing.T) { + r, call := rcNewRun(t, "operations/publiclink") + in := rc.Params{ + "fs": r.FremoteName, + "remote": "", + "expire": "5m", + "unlink": false, + } + _, err := call.Fn(context.Background(), in) + require.Error(t, err) + assert.Contains(t, err.Error(), "doesn't support public links") +} + +// operations/fsinfo: Return information about the remote +func TestRcFsInfo(t *testing.T) { + r, call := rcNewRun(t, "operations/fsinfo") + in := rc.Params{ + "fs": r.FremoteName, + } + got, err := call.Fn(context.Background(), in) + require.NoError(t, err) + want := operations.GetFsInfo(r.Fremote) + assert.Equal(t, want.Name, got["Name"]) + assert.Equal(t, want.Root, got["Root"]) + assert.Equal(t, want.String, got["String"]) + assert.Equal(t, float64(want.Precision), got["Precision"]) + var hashes []any + for _, hash := range want.Hashes { + hashes = append(hashes, hash) + } + assert.Equal(t, hashes, got["Hashes"]) + var features = map[string]any{} + for k, v := range want.Features { + features[k] = v + } + assert.Equal(t, features, got["Features"]) + +} + +// operations/uploadfile : Tests if upload file succeeds +func TestUploadFile(t *testing.T) { + r, call := rcNewRun(t, "operations/uploadfile") + ctx := context.Background() + + testFileName := "uploadfile-test.txt" + testFileContent := "Hello World" + r.WriteFile(testFileName, testFileContent, t1) + testItem1 := fstest.NewItem(testFileName, testFileContent, t1) + testItem2 := fstest.NewItem(path.Join("subdir", testFileName), testFileContent, t1) + + currentFile, err := os.Open(path.Join(r.LocalName, testFileName)) + require.NoError(t, err) + + defer func() { + assert.NoError(t, currentFile.Close()) + }() + + formReader, contentType, _, err := rest.MultipartUpload(ctx, currentFile, url.Values{}, "file", testFileName, "application/octet-stream") + require.NoError(t, err) + + httpReq := httptest.NewRequest("POST", "/", formReader) + httpReq.Header.Add("Content-Type", contentType) + + in := rc.Params{ + "_request": httpReq, + "fs": r.FremoteName, + "remote": "", + } + + _, err = call.Fn(context.Background(), in) + require.NoError(t, err) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{testItem1}, nil, fs.ModTimeNotSupported) + + assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir")) + + currentFile2, err := os.Open(path.Join(r.LocalName, testFileName)) + require.NoError(t, err) + + defer func() { + assert.NoError(t, currentFile2.Close()) + }() + + formReader, contentType, _, err = rest.MultipartUpload(ctx, currentFile2, url.Values{}, "file", testFileName, "application/octet-stream") + require.NoError(t, err) + + httpReq = httptest.NewRequest("POST", "/", formReader) + httpReq.Header.Add("Content-Type", contentType) + + in = rc.Params{ + "_request": httpReq, + "fs": r.FremoteName, + "remote": "subdir", + } + + _, err = call.Fn(context.Background(), in) + require.NoError(t, err) + + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{testItem1, testItem2}, nil, fs.ModTimeNotSupported) + +} + +// operations/command: Runs a backend command +func TestRcCommand(t *testing.T) { + r, call := rcNewRun(t, "backend/command") + in := rc.Params{ + "fs": r.FremoteName, + "command": "noop", + "opt": map[string]string{ + "echo": "true", + "blue": "", + }, + "arg": []string{ + "path1", + "path2", + }, + } + got, err := call.Fn(context.Background(), in) + if err != nil { + assert.False(t, r.Fremote.Features().IsLocal, "mustn't fail on local remote") + assert.Contains(t, err.Error(), "command not found") + return + } + want := rc.Params{"result": map[string]any{ + "arg": []string{ + "path1", + "path2", + }, + "name": "noop", + "opt": map[string]string{ + "blue": "", + "echo": "true", + }, + }} + assert.Equal(t, want, got) + errTxt := "explosion in the sausage factory" + in["opt"].(map[string]string)["error"] = errTxt + _, err = call.Fn(context.Background(), in) + assert.Error(t, err) + assert.Contains(t, err.Error(), errTxt) +} + +// operations/command: Runs a backend command +func TestRcDu(t *testing.T) { + ctx := context.Background() + _, call := rcNewRun(t, "core/du") + in := rc.Params{} + out, err := call.Fn(ctx, in) + if err == diskusage.ErrUnsupported { + t.Skip(err) + } + assert.NotEqual(t, "", out["dir"]) + info := out["info"].(diskusage.Info) + assert.True(t, info.Total != 0) + assert.True(t, info.Total > info.Free) + assert.True(t, info.Total > info.Available) + assert.True(t, info.Free >= info.Available) +} + +// operations/check: check the source and destination are the same +func TestRcCheck(t *testing.T) { + ctx := context.Background() + r, call := rcNewRun(t, "operations/check") + r.Mkdir(ctx, r.Fremote) + + MD5SUMS := ` +0ef726ce9b1a7692357ff70dd321d595 file1 +deadbeefcafe00000000000000000000 subdir/file2 +0386a8b8fcf672c326845c00ba41b9e2 subdir/subsubdir/file4 +` + + file1 := r.WriteBoth(ctx, "file1", "file1 contents", t1) + file2 := r.WriteFile("subdir/file2", MD5SUMS, t2) + file3 := r.WriteObject(ctx, "subdir/subsubdir/file3", "file3 contents", t3) + file4a := r.WriteFile("subdir/subsubdir/file4", "file4 contents", t3) + file4b := r.WriteObject(ctx, "subdir/subsubdir/file4", "file4 different contents", t3) + // operations.HashLister(ctx, hash.MD5, false, false, r.Fremote, os.Stdout) + + r.CheckLocalItems(t, file1, file2, file4a) + r.CheckRemoteItems(t, file1, file3, file4b) + + pstring := func(items ...fstest.Item) *[]string { + xs := make([]string, len(items)) + for i, item := range items { + xs[i] = item.Path + } + return &xs + } + + for _, testName := range []string{"Normal", "Download"} { + t.Run(testName, func(t *testing.T) { + in := rc.Params{ + "srcFs": r.LocalName, + "dstFs": r.FremoteName, + "combined": true, + "missingOnSrc": true, + "missingOnDst": true, + "match": true, + "differ": true, + "error": true, + } + if testName == "Download" { + in["download"] = true + } + out, err := call.Fn(ctx, in) + require.NoError(t, err) + + combined := []string{ + "= " + file1.Path, + "+ " + file2.Path, + "- " + file3.Path, + "* " + file4a.Path, + } + sort.Strings(combined) + sort.Strings(*out["combined"].(*[]string)) + want := rc.Params{ + "missingOnSrc": pstring(file3), + "missingOnDst": pstring(file2), + "differ": pstring(file4a), + "error": pstring(), + "match": pstring(file1), + "combined": &combined, + "status": "3 differences found", + "success": false, + } + if testName == "Normal" { + want["hashType"] = "md5" + } + + assert.Equal(t, want, out) + }) + } + + t.Run("CheckFile", func(t *testing.T) { + // The checksum file is treated as the source and srcFs is not used + in := rc.Params{ + "dstFs": r.FremoteName, + "combined": true, + "missingOnSrc": true, + "missingOnDst": true, + "match": true, + "differ": true, + "error": true, + "checkFileFs": r.LocalName, + "checkFileRemote": file2.Path, + "checkFileHash": "md5", + } + out, err := call.Fn(ctx, in) + require.NoError(t, err) + + combined := []string{ + "= " + file1.Path, + "+ " + file2.Path, + "- " + file3.Path, + "* " + file4a.Path, + } + sort.Strings(combined) + sort.Strings(*out["combined"].(*[]string)) + if strings.HasPrefix(out["status"].(string), "file not in") { + out["status"] = "file not in" + } + want := rc.Params{ + "missingOnSrc": pstring(file3), + "missingOnDst": pstring(file2), + "differ": pstring(file4a), + "error": pstring(), + "match": pstring(file1), + "combined": &combined, + "hashType": "md5", + "status": "file not in", + "success": false, + } + + assert.Equal(t, want, out) + }) + +} + +// operations/hashsum: hashsum a directory +func TestRcHashsum(t *testing.T) { + ctx := context.Background() + r, call := rcNewRun(t, "operations/hashsum") + r.Mkdir(ctx, r.Fremote) + + file1Contents := "file1 contents" + file1 := r.WriteBoth(ctx, "hashsum-file1", file1Contents, t1) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) + + hasher := hash.NewMultiHasher() + _, err := hasher.Write([]byte(file1Contents)) + require.NoError(t, err) + + for _, test := range []struct { + ht hash.Type + base64 bool + download bool + }{ + { + ht: r.Fremote.Hashes().GetOne(), + }, { + ht: r.Fremote.Hashes().GetOne(), + base64: true, + }, { + ht: hash.Whirlpool, + base64: false, + download: true, + }, { + ht: hash.Whirlpool, + base64: true, + download: true, + }, + } { + t.Run(fmt.Sprintf("hash=%v,base64=%v,download=%v", test.ht, test.base64, test.download), func(t *testing.T) { + file1Hash, err := hasher.SumString(test.ht, test.base64) + require.NoError(t, err) + + in := rc.Params{ + "fs": r.FremoteName, + "hashType": test.ht.String(), + "base64": test.base64, + "download": test.download, + } + + out, err := call.Fn(ctx, in) + require.NoError(t, err) + assert.Equal(t, test.ht.String(), out["hashType"]) + want := []string{ + fmt.Sprintf("%s hashsum-file1", file1Hash), + } + assert.Equal(t, want, out["hashsum"]) + }) + } +} + +// operations/hashsum: hashsum a single file +func TestRcHashsumSingleFile(t *testing.T) { + ctx := context.Background() + r, call := rcNewRun(t, "operations/hashsum") + r.Mkdir(ctx, r.Fremote) + + file1Contents := "file1 contents" + file1 := r.WriteBoth(ctx, "hashsum-file1", file1Contents, t1) + file2Contents := "file2 contents" + file2 := r.WriteBoth(ctx, "hashsum-file2", file2Contents, t1) + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file1, file2) + + // Make an fs pointing to just the file + fsString := path.Join(r.FremoteName, file1.Path) + + in := rc.Params{ + "fs": fsString, + "hashType": "MD5", + "download": true, + } + + out, err := call.Fn(ctx, in) + require.NoError(t, err) + assert.Equal(t, "md5", out["hashType"]) + assert.Equal(t, []string{"0ef726ce9b1a7692357ff70dd321d595 hashsum-file1"}, out["hashsum"]) +} + +// operations/hashsumfile: hashsum a single file +func TestRcHashsumFile(t *testing.T) { + ctx := context.Background() + r, call := rcNewRun(t, "operations/hashsumfile") + r.Mkdir(ctx, r.Fremote) + + file1Contents := "file1 contents" + file1 := r.WriteBoth(ctx, "hashsumfile-file1", file1Contents, t1) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) + + in := rc.Params{ + "fs": r.FremoteName, + "remote": file1.Path, + "hashType": "MD5", + "download": true, + } + + out, err := call.Fn(ctx, in) + require.NoError(t, err) + assert.Equal(t, "md5", out["hashType"]) + assert.Equal(t, "0ef726ce9b1a7692357ff70dd321d595", out["hash"]) +} diff --git a/fs/operations/reopen.go b/fs/operations/reopen.go new file mode 100644 index 0000000..fcc2a68 --- /dev/null +++ b/fs/operations/reopen.go @@ -0,0 +1,346 @@ +package operations + +import ( + "context" + "errors" + "io" + "sync" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/fserrors" +) + +// AccountFn is a function which will be called after every read +// from the ReOpen. +// +// It may return an error which will be passed back to the user. +type AccountFn func(n int) error + +// ReOpen is a wrapper for an object reader which reopens the stream on error +type ReOpen struct { + ctx context.Context + mu sync.Mutex // mutex to protect the below + readAtMu sync.Mutex // mutex to serialize the ReadAt calls + src fs.Object // object to open + baseOptions []fs.OpenOption // options to pass to initial open and where offset == 0 + options []fs.OpenOption // option to pass on subsequent opens where offset != 0 + rangeOption fs.RangeOption // adjust this range option on re-opens + rc io.ReadCloser // underlying stream + size int64 // total size of object - can be -ve + start int64 // absolute position to start reading from + end int64 // absolute position to end reading (exclusive) + offset int64 // offset in the file we are at, offset from start + newOffset int64 // if different to offset, reopen needed + maxTries int // maximum number of retries + tries int // number of retries we've had so far in this stream + err error // if this is set then Read/Close calls will return it + opened bool // if set then rc is valid and needs closing + account AccountFn // account for a read + reads int // count how many times the data has been read + accountOn int // only account on or after this read +} + +var ( + errFileClosed = errors.New("file already closed") + errTooManyTries = errors.New("failed to reopen: too many retries") + errInvalidWhence = errors.New("reopen Seek: invalid whence") + errNegativeSeek = errors.New("reopen Seek: negative position") + errSeekPastEnd = errors.New("reopen Seek: attempt to seek past end of data") + errBadEndSeek = errors.New("reopen Seek: can't seek from end with unknown sized object") +) + +// NewReOpen makes a handle which will reopen itself and seek to where +// it was on errors up to maxTries times. +// +// If an fs.HashesOption is set this will be applied when reading from +// the start. +// +// If an fs.RangeOption is set then this will applied when reading from +// the start, and updated on retries. +func NewReOpen(ctx context.Context, src fs.Object, maxTries int, options ...fs.OpenOption) (rc *ReOpen, err error) { + h := &ReOpen{ + ctx: ctx, + src: src, + maxTries: maxTries, + baseOptions: options, + size: src.Size(), + start: 0, + offset: 0, + newOffset: -1, // -1 means no seek required + } + h.mu.Lock() + defer h.mu.Unlock() + + // Filter the options for subsequent opens + h.options = make([]fs.OpenOption, 0, len(options)+1) + var limit int64 = -1 + for _, option := range options { + switch x := option.(type) { + case *fs.HashesOption: + // leave hash option out when ranging + case *fs.RangeOption: + h.start, limit = x.Decode(h.end) + case *fs.SeekOption: + h.start, limit = x.Offset, -1 + default: + h.options = append(h.options, option) + } + } + + // Put our RangeOption on the end + h.rangeOption.Start = h.start + h.options = append(h.options, &h.rangeOption) + + // If a size range is set then set the end point of the file to that + if limit >= 0 && h.size >= 0 { + h.end = h.start + limit + h.rangeOption.End = h.end - 1 // remember range options are inclusive + } else { + h.end = h.size + h.rangeOption.End = -1 + } + + err = h.open() + if err != nil { + return nil, err + } + return h, nil +} + +// Open makes a handle which will reopen itself and seek to where it +// was on errors. +// +// If an fs.HashesOption is set this will be applied when reading from +// the start. +// +// If an fs.RangeOption is set then this will applied when reading from +// the start, and updated on retries. +// +// It will obey LowLevelRetries in the ctx as the maximum number of +// tries. +// +// Use this instead of calling the Open method on fs.Objects +func Open(ctx context.Context, src fs.Object, options ...fs.OpenOption) (rc *ReOpen, err error) { + maxTries := fs.GetConfig(ctx).LowLevelRetries + return NewReOpen(ctx, src, maxTries, options...) +} + +// open the underlying handle - call with lock held +// +// we don't retry here as the Open() call will itself have low level retries +func (h *ReOpen) open() error { + var opts []fs.OpenOption + if h.offset == 0 { + // if reading from the start using the initial options + opts = h.baseOptions + } else { + // otherwise use the filtered options + opts = h.options + // Adjust range start to where we have got to + h.rangeOption.Start = h.start + h.offset + } + // Make a copy of the options as fs.FixRangeOption modifies them :-( + opts = append(make([]fs.OpenOption, 0, len(opts)), opts...) + h.tries++ + if h.tries > h.maxTries { + h.err = errTooManyTries + } else { + h.rc, h.err = h.src.Open(h.ctx, opts...) + } + if h.err != nil { + if h.tries > 1 { + fs.Debugf(h.src, "Reopen failed after offset %d bytes read: %v", h.offset, h.err) + } + return h.err + } + h.opened = true + return nil +} + +// reopen the underlying handle by closing it and reopening it. +func (h *ReOpen) reopen() (err error) { + // close underlying stream if needed + if h.opened { + h.opened = false + _ = h.rc.Close() + } + return h.open() +} + +// account for n bytes being read +func (h *ReOpen) accountRead(n int) error { + if h.account == nil { + return nil + } + // Don't start accounting until we've reached this many reads + // + // rw.reads will be 1 the first time this is called + // rw.accountOn 2 means start accounting on the 2nd read through + if h.reads >= h.accountOn { + return h.account(n) + } + return nil +} + +// Read bytes retrying as necessary +func (h *ReOpen) Read(p []byte) (n int, err error) { + h.mu.Lock() + defer h.mu.Unlock() + if h.err != nil { + // return a previous error if there is one + return n, h.err + } + + // re-open if seek needed + if h.newOffset >= 0 { + if h.offset != h.newOffset { + fs.Debugf(h.src, "Seek from %d to %d", h.offset, h.newOffset) + h.offset = h.newOffset + err = h.reopen() + if err != nil { + return 0, err + } + } + h.newOffset = -1 + } + + // Read a full buffer + startOffset := h.offset + var nn int + for n < len(p) && err == nil { + nn, err = h.rc.Read(p[n:]) + n += nn + h.offset += int64(nn) + if err != nil && err != io.EOF { + h.err = err + if !fserrors.IsNoLowLevelRetryError(err) { + fs.Debugf(h.src, "Reopening on read failure after offset %d bytes: retry %d/%d: %v", h.offset, h.tries, h.maxTries, err) + if h.reopen() == nil { + err = nil + } + } + } + } + // Count a read of the data if we read from the start successfully + if startOffset == 0 && n != 0 { + h.reads++ + } + // Account the read + accErr := h.accountRead(n) + if err == nil { + err = accErr + } + return n, err +} + +// ReadAt reads len(p) bytes at absolute offset off without changing +// the read position. +// +// Note: operations are serialized; it won't behave like a truly +// concurrent ReaderAt. +func (h *ReOpen) ReadAt(p []byte, off int64) (n int, err error) { + h.readAtMu.Lock() + defer h.readAtMu.Unlock() + + // Save current position + cur, err := h.Seek(0, io.SeekCurrent) + if err != nil { + return 0, err + } + // Seek to requested offset + if _, err = h.Seek(off, io.SeekStart); err != nil { + return 0, err + } + // Restore position on exit + defer func() { + if _, seekErr := h.Seek(cur, io.SeekStart); seekErr != nil && err == nil { + err = seekErr + } + }() + + // Fill p fully unless EOF + return h.Read(p) +} + +// Seek sets the offset for the next Read or Write to offset, interpreted +// according to whence: SeekStart means relative to the start of the file, +// SeekCurrent means relative to the current offset, and SeekEnd means relative +// to the end (for example, offset = -2 specifies the penultimate byte of the +// file). Seek returns the new offset relative to the start of the file or an +// error, if any. +// +// Seeking to an offset before the start of the file is an error. Seeking +// to any positive offset may be allowed, but if the new offset exceeds the +// size of the underlying object the behavior of subsequent I/O operations is +// implementation-dependent. +func (h *ReOpen) Seek(offset int64, whence int) (int64, error) { + h.mu.Lock() + defer h.mu.Unlock() + if h.err != nil { + // return a previous error if there is one + return 0, h.err + } + var abs int64 + var size = h.end - h.start + switch whence { + case io.SeekStart: + abs = offset + case io.SeekCurrent: + if h.newOffset >= 0 { + abs = h.newOffset + offset + } else { + abs = h.offset + offset + } + case io.SeekEnd: + if h.size < 0 { + return 0, errBadEndSeek + } + abs = size + offset + default: + return 0, errInvalidWhence + } + if abs < 0 { + return 0, errNegativeSeek + } + if h.size >= 0 && abs > size { + return size, errSeekPastEnd + } + + h.tries = 0 // Reset open count on seek + h.newOffset = abs // New offset - applied in Read + return abs, nil +} + +// Close the stream +func (h *ReOpen) Close() error { + h.mu.Lock() + defer h.mu.Unlock() + if !h.opened { + return errFileClosed + } + h.opened = false + h.err = errFileClosed + return h.rc.Close() +} + +// SetAccounting should be provided with a function which will be +// called after every read from the RW. +// +// It may return an error which will be passed back to the user. +func (h *ReOpen) SetAccounting(account AccountFn) *ReOpen { + h.account = account + return h +} + +// DelayAccounting makes sure the accounting function only gets called +// on the i-th or later read of the data from this point (counting +// from 1). +// +// This is useful so that we don't account initial reads of the data +// e.g. when calculating hashes. +// +// Set this to 0 to account everything. +func (h *ReOpen) DelayAccounting(i int) { + h.accountOn = i + h.reads = 0 +} diff --git a/fs/operations/reopen_test.go b/fs/operations/reopen_test.go new file mode 100644 index 0000000..19f6ca3 --- /dev/null +++ b/fs/operations/reopen_test.go @@ -0,0 +1,429 @@ +package operations + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fstest/mockobject" + "github.com/rclone/rclone/lib/pool" + "github.com/rclone/rclone/lib/readers" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// check interfaces +var ( + _ io.ReadSeekCloser = (*ReOpen)(nil) + _ io.ReaderAt = (*ReOpen)(nil) + _ pool.DelayAccountinger = (*ReOpen)(nil) +) + +var errorTestError = errors.New("test error") + +// this is a wrapper for a mockobject with a custom Open function +// +// breaks indicate the number of bytes to read before returning an +// error +type reOpenTestObject struct { + fs.Object + t *testing.T + wantStart int64 + breaks []int64 + unknownSize bool +} + +// Open opens the file for read. Call Close() on the returned io.ReadCloser +// +// This will break after reading the number of bytes in breaks +func (o *reOpenTestObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { + // Lots of backends do this - make sure it works as it modifies options + fs.FixRangeOption(options, o.Size()) + gotHash := false + gotRange := false + startPos := int64(0) + for _, option := range options { + switch x := option.(type) { + case *fs.HashesOption: + gotHash = true + case *fs.RangeOption: + gotRange = true + startPos = x.Start + if o.unknownSize { + assert.Equal(o.t, int64(-1), x.End) + } + case *fs.SeekOption: + startPos = x.Offset + } + } + assert.Equal(o.t, o.wantStart, startPos) + // Check if ranging, mustn't have hash if offset != 0 + if gotHash && gotRange { + assert.Equal(o.t, int64(0), startPos) + } + rc, err := o.Object.Open(ctx, options...) + if err != nil { + return nil, err + } + if len(o.breaks) > 0 { + // Pop a breakpoint off + N := o.breaks[0] + o.breaks = o.breaks[1:] + o.wantStart += N + // If 0 then return an error immediately + if N == 0 { + return nil, errorTestError + } + // Read N bytes then an error + r := io.MultiReader(&io.LimitedReader{R: rc, N: N}, readers.ErrorReader{Err: errorTestError}) + // Wrap with Close in a new readCloser + rc = readCloser{Reader: r, Closer: rc} + } + return rc, nil +} + +func TestReOpen(t *testing.T) { + for _, testName := range []string{"Normal", "WithRangeOption", "WithSeekOption", "UnknownSize"} { + t.Run(testName, func(t *testing.T) { + // Contents for the mock object + var ( + reOpenTestcontents = []byte("0123456789") + expectedRead = reOpenTestcontents + rangeOption *fs.RangeOption + seekOption *fs.SeekOption + unknownSize = false + ) + switch testName { + case "Normal": + case "WithRangeOption": + rangeOption = &fs.RangeOption{Start: 1, End: 7} // range is inclusive + expectedRead = reOpenTestcontents[1:8] + case "WithSeekOption": + seekOption = &fs.SeekOption{Offset: 2} + expectedRead = reOpenTestcontents[2:] + case "UnknownSize": + rangeOption = &fs.RangeOption{Start: 1, End: -1} + expectedRead = reOpenTestcontents[1:] + unknownSize = true + default: + panic("bad test name") + } + + // Start the test with the given breaks + testReOpen := func(breaks []int64, maxRetries int) (*ReOpen, *reOpenTestObject, error) { + srcOrig := mockobject.New("potato").WithContent(reOpenTestcontents, mockobject.SeekModeNone) + srcOrig.SetUnknownSize(unknownSize) + src := &reOpenTestObject{ + Object: srcOrig, + t: t, + breaks: breaks, + unknownSize: unknownSize, + } + opts := []fs.OpenOption{} + if rangeOption == nil && seekOption == nil { + opts = append(opts, &fs.HashesOption{Hashes: hash.NewHashSet(hash.MD5)}) + } + if rangeOption != nil { + opts = append(opts, rangeOption) + src.wantStart = rangeOption.Start + } + if seekOption != nil { + opts = append(opts, seekOption) + src.wantStart = seekOption.Offset + } + rc, err := NewReOpen(context.Background(), src, maxRetries, opts...) + return rc, src, err + } + + // Reset the start after a seek, taking into account the offset + setWantStart := func(src *reOpenTestObject, x int64) { + src.wantStart = x + if rangeOption != nil { + src.wantStart += rangeOption.Start + } else if seekOption != nil { + src.wantStart += seekOption.Offset + } + } + + t.Run("Basics", func(t *testing.T) { + // open + h, _, err := testReOpen(nil, 10) + assert.NoError(t, err) + + // Check contents read correctly + got, err := io.ReadAll(h) + assert.NoError(t, err) + assert.Equal(t, expectedRead, got) + + // Check read after end + var buf = make([]byte, 1) + n, err := h.Read(buf) + assert.Equal(t, 0, n) + assert.Equal(t, io.EOF, err) + + // Rewind the stream + _, err = h.Seek(0, io.SeekStart) + require.NoError(t, err) + + // Check contents read correctly + got, err = io.ReadAll(h) + assert.NoError(t, err) + assert.Equal(t, expectedRead, got) + + // Check close + assert.NoError(t, h.Close()) + + // Check double close + assert.Equal(t, errFileClosed, h.Close()) + + // Check read after close + n, err = h.Read(buf) + assert.Equal(t, 0, n) + assert.Equal(t, errFileClosed, err) + }) + + t.Run("ErrorAtStart", func(t *testing.T) { + // open with immediate breaking + h, _, err := testReOpen([]int64{0}, 10) + assert.Equal(t, errorTestError, err) + assert.Nil(t, h) + }) + + t.Run("WithErrors", func(t *testing.T) { + // open with a few break points but less than the max + h, _, err := testReOpen([]int64{2, 1, 3}, 10) + assert.NoError(t, err) + + // check contents + got, err := io.ReadAll(h) + assert.NoError(t, err) + assert.Equal(t, expectedRead, got) + + // check close + assert.NoError(t, h.Close()) + }) + + t.Run("TooManyErrors", func(t *testing.T) { + // open with a few break points but >= the max + h, _, err := testReOpen([]int64{2, 1, 3}, 3) + assert.NoError(t, err) + + // check contents + got, err := io.ReadAll(h) + assert.Equal(t, errorTestError, err) + assert.Equal(t, expectedRead[:6], got) + + // check old error is returned + var buf = make([]byte, 1) + n, err := h.Read(buf) + assert.Equal(t, 0, n) + assert.Equal(t, errTooManyTries, err) + + // Check close + assert.Equal(t, errFileClosed, h.Close()) + }) + + t.Run("ReadAt", func(t *testing.T) { + // open + h, src, err := testReOpen([]int64{2, 1, 3}, 10) + assert.NoError(t, err) + + buf := make([]byte, 5) + + // Read at 0 + n, err := h.ReadAt(buf, 0) + require.NoError(t, err) + assert.Equal(t, 5, n) + assert.Equal(t, expectedRead[:n], buf[:n]) + + // Read at 1 + setWantStart(src, 1) + n, err = h.ReadAt(buf[:3], 1) + require.NoError(t, err) + assert.Equal(t, 3, n) + assert.Equal(t, expectedRead[1:n+1], buf[:n]) + + // check position unchanged + pos, err := h.Seek(0, io.SeekCurrent) + require.NoError(t, err) + assert.Equal(t, int64(0), pos) + + // check close + assert.NoError(t, h.Close()) + _, err = h.Seek(0, io.SeekCurrent) + assert.Equal(t, errFileClosed, err) + }) + + t.Run("Seek", func(t *testing.T) { + // open + h, src, err := testReOpen([]int64{2, 1, 3}, 10) + assert.NoError(t, err) + + // Seek to end + pos, err := h.Seek(int64(len(expectedRead)), io.SeekStart) + assert.NoError(t, err) + assert.Equal(t, int64(len(expectedRead)), pos) + + // Seek to start + pos, err = h.Seek(0, io.SeekStart) + assert.NoError(t, err) + assert.Equal(t, int64(0), pos) + + // Should not allow seek past end + pos, err = h.Seek(int64(len(expectedRead))+1, io.SeekCurrent) + if !unknownSize { + assert.Equal(t, errSeekPastEnd, err) + assert.Equal(t, len(expectedRead), int(pos)) + } else { + assert.Equal(t, nil, err) + assert.Equal(t, len(expectedRead)+1, int(pos)) + + // Seek back to start to get tests in sync + pos, err = h.Seek(0, io.SeekStart) + assert.NoError(t, err) + assert.Equal(t, int64(0), pos) + } + + // Should not allow seek to negative position start + pos, err = h.Seek(-1, io.SeekCurrent) + assert.Equal(t, errNegativeSeek, err) + assert.Equal(t, 0, int(pos)) + + // Should not allow seek with invalid whence + pos, err = h.Seek(0, 3) + assert.Equal(t, errInvalidWhence, err) + assert.Equal(t, 0, int(pos)) + + // check read + dst := make([]byte, 5) + n, err := h.Read(dst) + assert.Nil(t, err) + assert.Equal(t, 5, n) + assert.Equal(t, expectedRead[:5], dst) + + // Test io.SeekCurrent + pos, err = h.Seek(-3, io.SeekCurrent) + assert.Nil(t, err) + assert.Equal(t, 2, int(pos)) + + // check read + setWantStart(src, 2) + n, err = h.Read(dst) + assert.Nil(t, err) + assert.Equal(t, 5, n) + assert.Equal(t, expectedRead[2:7], dst) + + pos, err = h.Seek(-2, io.SeekCurrent) + assert.Nil(t, err) + assert.Equal(t, 5, int(pos)) + + // Test io.SeekEnd + pos, err = h.Seek(-3, io.SeekEnd) + if !unknownSize { + assert.Nil(t, err) + assert.Equal(t, len(expectedRead)-3, int(pos)) + } else { + assert.Equal(t, errBadEndSeek, err) + assert.Equal(t, 0, int(pos)) + + // sync + pos, err = h.Seek(1, io.SeekCurrent) + assert.Nil(t, err) + assert.Equal(t, 6, int(pos)) + } + + // check read + dst = make([]byte, 3) + setWantStart(src, int64(len(expectedRead)-3)) + n, err = h.Read(dst) + assert.Nil(t, err) + assert.Equal(t, 3, n) + assert.Equal(t, expectedRead[len(expectedRead)-3:], dst) + + // check close + assert.NoError(t, h.Close()) + _, err = h.Seek(0, io.SeekCurrent) + assert.Equal(t, errFileClosed, err) + }) + + t.Run("AccountRead", func(t *testing.T) { + h, _, err := testReOpen(nil, 10) + assert.NoError(t, err) + + var total int + h.SetAccounting(func(n int) error { + total += n + return nil + }) + + dst := make([]byte, 3) + n, err := h.Read(dst) + assert.Equal(t, 3, n) + assert.NoError(t, err) + assert.Equal(t, 3, total) + }) + + t.Run("AccountReadDelay", func(t *testing.T) { + h, _, err := testReOpen(nil, 10) + assert.NoError(t, err) + + var total int + h.SetAccounting(func(n int) error { + total += n + return nil + }) + + rewind := func() { + _, err := h.Seek(0, io.SeekStart) + require.NoError(t, err) + } + + h.DelayAccounting(3) + + dst := make([]byte, 16) + + n, err := h.Read(dst) + assert.Equal(t, len(expectedRead), n) + assert.Equal(t, io.EOF, err) + assert.Equal(t, 0, total) + rewind() + + n, err = h.Read(dst) + assert.Equal(t, len(expectedRead), n) + assert.Equal(t, io.EOF, err) + assert.Equal(t, 0, total) + rewind() + + n, err = h.Read(dst) + assert.Equal(t, len(expectedRead), n) + assert.Equal(t, io.EOF, err) + assert.Equal(t, len(expectedRead), total) + rewind() + + n, err = h.Read(dst) + assert.Equal(t, len(expectedRead), n) + assert.Equal(t, io.EOF, err) + assert.Equal(t, 2*len(expectedRead), total) + rewind() + }) + + t.Run("AccountReadError", func(t *testing.T) { + // Test accounting errors + h, _, err := testReOpen(nil, 10) + assert.NoError(t, err) + + h.SetAccounting(func(n int) error { + return errorTestError + }) + + dst := make([]byte, 3) + n, err := h.Read(dst) + assert.Equal(t, 3, n) + assert.Equal(t, errorTestError, err) + }) + }) + } +} diff --git a/fs/sync/pipe.go b/fs/sync/pipe.go new file mode 100644 index 0000000..124a077 --- /dev/null +++ b/fs/sync/pipe.go @@ -0,0 +1,237 @@ +package sync + +import ( + "context" + "fmt" + "math/bits" + "strconv" + "strings" + "sync" + + "github.com/aalpar/deheap" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/fserrors" +) + +// compare two items for order by +type lessFn func(a, b fs.ObjectPair) bool + +// pipe provides an unbounded channel like experience +// +// Note unlike channels these aren't strictly ordered. +type pipe struct { + mu sync.Mutex + c chan struct{} + queue []fs.ObjectPair + closed bool + totalSize int64 + stats func(items int, totalSize int64) + less lessFn + fraction int +} + +func newPipe(orderBy string, stats func(items int, totalSize int64), maxBacklog int) (*pipe, error) { + if maxBacklog < 0 { + maxBacklog = (1 << (bits.UintSize - 1)) - 1 // largest positive int + } + less, fraction, err := newLess(orderBy) + if err != nil { + return nil, fserrors.FatalError(err) + } + p := &pipe{ + c: make(chan struct{}, maxBacklog), + stats: stats, + less: less, + fraction: fraction, + } + if p.less != nil { + deheap.Init(p) + } + return p, nil +} + +// Len satisfy heap.Interface - must be called with lock held +func (p *pipe) Len() int { + return len(p.queue) +} + +// Len satisfy heap.Interface - must be called with lock held +func (p *pipe) Less(i, j int) bool { + return p.less(p.queue[i], p.queue[j]) +} + +// Swap satisfy heap.Interface - must be called with lock held +func (p *pipe) Swap(i, j int) { + p.queue[i], p.queue[j] = p.queue[j], p.queue[i] +} + +// Push satisfy heap.Interface - must be called with lock held +func (p *pipe) Push(item any) { + p.queue = append(p.queue, item.(fs.ObjectPair)) +} + +// Pop satisfy heap.Interface - must be called with lock held +func (p *pipe) Pop() any { + old := p.queue + n := len(old) + item := old[n-1] + old[n-1] = fs.ObjectPair{} // avoid memory leak + p.queue = old[0 : n-1] + return item +} + +// Put a pair into the pipe +// +// It returns ok = false if the context was cancelled +// +// It will panic if you call it after Close() +// +// Note that pairs where src==dst aren't counted for stats +func (p *pipe) Put(ctx context.Context, pair fs.ObjectPair) (ok bool) { + if ctx.Err() != nil { + return false + } + p.mu.Lock() + if p.less == nil { + // no order-by + p.queue = append(p.queue, pair) + } else { + deheap.Push(p, pair) + } + size := pair.Src.Size() + if size > 0 && pair.Src != pair.Dst { + p.totalSize += size + } + p.stats(len(p.queue), p.totalSize) + p.mu.Unlock() + select { + case <-ctx.Done(): + return false + case p.c <- struct{}{}: + } + return true +} + +// Get a pair from the pipe +// +// If fraction is > the mixed fraction set in the pipe then it gets it +// from the other end of the heap if order-by is in effect +// +// It returns ok = false if the context was cancelled or Close() has +// been called. +func (p *pipe) GetMax(ctx context.Context, fraction int) (pair fs.ObjectPair, ok bool) { + if ctx.Err() != nil { + return + } + select { + case <-ctx.Done(): + return + case _, ok = <-p.c: + if !ok { + return + } + } + p.mu.Lock() + if p.less == nil { + // no order-by + pair = p.queue[0] + p.queue[0] = fs.ObjectPair{} // avoid memory leak + p.queue = p.queue[1:] + } else if p.fraction < 0 || fraction < p.fraction { + pair = deheap.Pop(p).(fs.ObjectPair) + } else { + pair = deheap.PopMax(p).(fs.ObjectPair) + } + size := pair.Src.Size() + if size > 0 && pair.Src != pair.Dst { + p.totalSize -= size + } + if p.totalSize < 0 { + p.totalSize = 0 + } + p.stats(len(p.queue), p.totalSize) + p.mu.Unlock() + return pair, true +} + +// Get a pair from the pipe +// +// It returns ok = false if the context was cancelled or Close() has +// been called. +func (p *pipe) Get(ctx context.Context) (pair fs.ObjectPair, ok bool) { + return p.GetMax(ctx, -1) +} + +// Stats reads the number of items in the queue and the totalSize +func (p *pipe) Stats() (items int, totalSize int64) { + p.mu.Lock() + items, totalSize = len(p.queue), p.totalSize + p.mu.Unlock() + return items, totalSize +} + +// Close the pipe +// +// Writes to a closed pipe will panic as will double closing a pipe +func (p *pipe) Close() { + p.mu.Lock() + close(p.c) + p.closed = true + p.mu.Unlock() +} + +// newLess returns a less function for the heap comparison or nil if +// one is not required +func newLess(orderBy string) (less lessFn, fraction int, err error) { + fraction = -1 + if orderBy == "" { + return nil, fraction, nil + } + parts := strings.Split(strings.ToLower(orderBy), ",") + switch parts[0] { + case "name": + less = func(a, b fs.ObjectPair) bool { + return a.Src.Remote() < b.Src.Remote() + } + case "size": + less = func(a, b fs.ObjectPair) bool { + return a.Src.Size() < b.Src.Size() + } + case "modtime": + less = func(a, b fs.ObjectPair) bool { + ctx := context.Background() + return a.Src.ModTime(ctx).Before(b.Src.ModTime(ctx)) + } + default: + return nil, fraction, fmt.Errorf("unknown --order-by comparison %q", parts[0]) + } + descending := false + if len(parts) > 1 { + switch parts[1] { + case "ascending", "asc": + case "descending", "desc": + descending = true + case "mixed": + fraction = 50 + if len(parts) > 2 { + fraction, err = strconv.Atoi(parts[2]) + if err != nil { + return nil, fraction, fmt.Errorf("bad mixed fraction --order-by %q", parts[2]) + } + } + + default: + return nil, fraction, fmt.Errorf("unknown --order-by sort direction %q", parts[1]) + } + } + if (fraction >= 0 && len(parts) > 3) || (fraction < 0 && len(parts) > 2) { + return nil, fraction, fmt.Errorf("bad --order-by string %q", orderBy) + } + if descending { + oldLess := less + less = func(a, b fs.ObjectPair) bool { + return !oldLess(a, b) + } + } + return less, fraction, nil +} diff --git a/fs/sync/pipe_test.go b/fs/sync/pipe_test.go new file mode 100644 index 0000000..94916aa --- /dev/null +++ b/fs/sync/pipe_test.go @@ -0,0 +1,290 @@ +package sync + +import ( + "container/heap" + "context" + "sync" + "sync/atomic" + "testing" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fstest/mockobject" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Check interface satisfied +var _ heap.Interface = (*pipe)(nil) + +func TestPipe(t *testing.T) { + var queueLength int + var queueSize int64 + stats := func(n int, size int64) { + queueLength, queueSize = n, size + } + + // Make a new pipe + p, err := newPipe("", stats, 10) + require.NoError(t, err) + + checkStats := func(expectedN int, expectedSize int64) { + n, size := p.Stats() + assert.Equal(t, expectedN, n) + assert.Equal(t, expectedSize, size) + assert.Equal(t, expectedN, queueLength) + assert.Equal(t, expectedSize, queueSize) + } + + checkStats(0, 0) + + ctx := context.Background() + + obj1 := mockobject.New("potato").WithContent([]byte("hello"), mockobject.SeekModeNone) + + pair1 := fs.ObjectPair{Src: obj1, Dst: nil} + pairD := fs.ObjectPair{Src: obj1, Dst: obj1} // this object should not count to the stats + + // Put an object + ok := p.Put(ctx, pair1) + assert.Equal(t, true, ok) + checkStats(1, 5) + + // Put an object to be deleted + ok = p.Put(ctx, pairD) + assert.Equal(t, true, ok) + checkStats(2, 5) + + // Close the pipe showing reading on closed pipe is OK + p.Close() + + // Read from pipe + pair2, ok := p.Get(ctx) + assert.Equal(t, pair1, pair2) + assert.Equal(t, true, ok) + checkStats(1, 0) + + // Read from pipe + pair2, ok = p.Get(ctx) + assert.Equal(t, pairD, pair2) + assert.Equal(t, true, ok) + checkStats(0, 0) + + // Check read on closed pipe + pair2, ok = p.Get(ctx) + assert.Equal(t, fs.ObjectPair{}, pair2) + assert.Equal(t, false, ok) + + // Check panic on write to closed pipe + assert.Panics(t, func() { p.Put(ctx, pair1) }) + + // Make a new pipe + p, err = newPipe("", stats, 10) + require.NoError(t, err) + ctx2, cancel := context.WithCancel(ctx) + + // cancel it in the background - check read ceases + go cancel() + pair2, ok = p.Get(ctx2) + assert.Equal(t, fs.ObjectPair{}, pair2) + assert.Equal(t, false, ok) + + // check we can't write + ok = p.Put(ctx2, pair1) + assert.Equal(t, false, ok) + +} + +// TestPipeConcurrent runs concurrent Get and Put to flush out any +// race conditions and concurrency problems. +func TestPipeConcurrent(t *testing.T) { + const ( + N = 1000 + readWriters = 10 + ) + + stats := func(n int, size int64) {} + + // Make a new pipe + p, err := newPipe("", stats, 10) + require.NoError(t, err) + + var wg sync.WaitGroup + obj1 := mockobject.New("potato").WithContent([]byte("hello"), mockobject.SeekModeNone) + pair1 := fs.ObjectPair{Src: obj1, Dst: nil} + ctx := context.Background() + var count atomic.Int64 + + for range readWriters { + wg.Add(2) + go func() { + defer wg.Done() + for range N { + // Read from pipe + pair2, ok := p.Get(ctx) + assert.Equal(t, pair1, pair2) + assert.Equal(t, true, ok) + count.Add(-1) + } + }() + go func() { + defer wg.Done() + for range N { + // Put an object + ok := p.Put(ctx, pair1) + assert.Equal(t, true, ok) + count.Add(1) + } + }() + } + wg.Wait() + + assert.Equal(t, int64(0), count.Load()) +} + +func TestPipeOrderBy(t *testing.T) { + var ( + stats = func(n int, size int64) {} + ctx = context.Background() + obj1 = mockobject.New("b").WithContent([]byte("1"), mockobject.SeekModeNone) + obj2 = mockobject.New("a").WithContent([]byte("22"), mockobject.SeekModeNone) + pair1 = fs.ObjectPair{Src: obj1} + pair2 = fs.ObjectPair{Src: obj2} + ) + + for _, test := range []struct { + orderBy string + swapped1 bool + swapped2 bool + fraction int + }{ + {"", false, true, -1}, + {"size", false, false, -1}, + {"name", true, true, -1}, + {"modtime", false, true, -1}, + {"size,ascending", false, false, -1}, + {"name,asc", true, true, -1}, + {"modtime,ascending", false, true, -1}, + {"size,descending", true, true, -1}, + {"name,desc", false, false, -1}, + {"modtime,descending", true, false, -1}, + {"size,mixed,50", false, false, 25}, + {"size,mixed,51", true, true, 75}, + } { + t.Run(test.orderBy, func(t *testing.T) { + p, err := newPipe(test.orderBy, stats, 10) + require.NoError(t, err) + + readAndCheck := func(swapped bool) { + var readFirst, readSecond fs.ObjectPair + var ok1, ok2 bool + if test.fraction < 0 { + readFirst, ok1 = p.Get(ctx) + readSecond, ok2 = p.Get(ctx) + } else { + readFirst, ok1 = p.GetMax(ctx, test.fraction) + readSecond, ok2 = p.GetMax(ctx, test.fraction) + } + assert.True(t, ok1) + assert.True(t, ok2) + + if swapped { + assert.True(t, readFirst == pair2 && readSecond == pair1) + } else { + assert.True(t, readFirst == pair1 && readSecond == pair2) + } + } + + ok := p.Put(ctx, pair1) + assert.True(t, ok) + ok = p.Put(ctx, pair2) + assert.True(t, ok) + + readAndCheck(test.swapped1) + + // insert other way round + + ok = p.Put(ctx, pair2) + assert.True(t, ok) + ok = p.Put(ctx, pair1) + assert.True(t, ok) + + readAndCheck(test.swapped2) + }) + } +} + +func TestNewLess(t *testing.T) { + t.Run("blankOK", func(t *testing.T) { + less, _, err := newLess("") + require.NoError(t, err) + assert.Nil(t, less) + }) + + t.Run("tooManyParts", func(t *testing.T) { + _, _, err := newLess("size,asc,toomanyparts") + require.Error(t, err) + assert.Contains(t, err.Error(), "bad --order-by string") + }) + + t.Run("tooManyParts2", func(t *testing.T) { + _, _, err := newLess("size,mixed,50,toomanyparts") + require.Error(t, err) + assert.Contains(t, err.Error(), "bad --order-by string") + }) + + t.Run("badMixed", func(t *testing.T) { + _, _, err := newLess("size,mixed,32.7") + require.Error(t, err) + assert.Contains(t, err.Error(), "bad mixed fraction") + }) + + t.Run("unknownComparison", func(t *testing.T) { + _, _, err := newLess("potato") + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown --order-by comparison") + }) + + t.Run("unknownSortDirection", func(t *testing.T) { + _, _, err := newLess("name,sideways") + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown --order-by sort direction") + }) + + var ( + obj1 = mockobject.New("b").WithContent([]byte("1"), mockobject.SeekModeNone) + obj2 = mockobject.New("a").WithContent([]byte("22"), mockobject.SeekModeNone) + pair1 = fs.ObjectPair{Src: obj1} + pair2 = fs.ObjectPair{Src: obj2} + ) + + for _, test := range []struct { + orderBy string + pair1LessPair2 bool + pair2LessPair1 bool + wantFraction int + }{ + {"size", true, false, -1}, + {"name", false, true, -1}, + {"modtime", false, false, -1}, + {"size,ascending", true, false, -1}, + {"name,asc", false, true, -1}, + {"modtime,ascending", false, false, -1}, + {"size,descending", false, true, -1}, + {"name,desc", true, false, -1}, + {"modtime,descending", true, true, -1}, + {"modtime,mixed", false, false, 50}, + {"modtime,mixed,30", false, false, 30}, + } { + t.Run(test.orderBy, func(t *testing.T) { + less, gotFraction, err := newLess(test.orderBy) + assert.Equal(t, test.wantFraction, gotFraction) + require.NoError(t, err) + require.NotNil(t, less) + pair1LessPair2 := less(pair1, pair2) + assert.Equal(t, test.pair1LessPair2, pair1LessPair2) + pair2LessPair1 := less(pair2, pair1) + assert.Equal(t, test.pair2LessPair1, pair2LessPair1) + }) + } + +} diff --git a/fs/sync/rc.go b/fs/sync/rc.go new file mode 100644 index 0000000..6577e49 --- /dev/null +++ b/fs/sync/rc.go @@ -0,0 +1,61 @@ +package sync + +import ( + "context" + + "github.com/rclone/rclone/fs/rc" +) + +func init() { + for _, name := range []string{"sync", "copy", "move"} { + moveHelp := "" + if name == "move" { + moveHelp = "- deleteEmptySrcDirs - delete empty src directories if set\n" + } + rc.Add(rc.Call{ + Path: "sync/" + name, + AuthRequired: true, + Fn: func(ctx context.Context, in rc.Params) (rc.Params, error) { + return rcSyncCopyMove(ctx, in, name) + }, + Title: name + " a directory from source remote to destination remote", + Help: `This takes the following parameters: + +- srcFs - a remote name string e.g. "drive:src" for the source +- dstFs - a remote name string e.g. "drive:dst" for the destination +- createEmptySrcDirs - create empty src directories on destination if set +` + moveHelp + ` + +See the [` + name + `](/commands/rclone_` + name + `/) command for more information on the above.`, + }) + } +} + +// Sync/Copy/Move a file +func rcSyncCopyMove(ctx context.Context, in rc.Params, name string) (out rc.Params, err error) { + srcFs, err := rc.GetFsNamed(ctx, in, "srcFs") + if err != nil { + return nil, err + } + dstFs, err := rc.GetFsNamed(ctx, in, "dstFs") + if err != nil { + return nil, err + } + createEmptySrcDirs, err := in.GetBool("createEmptySrcDirs") + if rc.NotErrParamNotFound(err) { + return nil, err + } + switch name { + case "sync": + return nil, Sync(ctx, dstFs, srcFs, createEmptySrcDirs) + case "copy": + return nil, CopyDir(ctx, dstFs, srcFs, createEmptySrcDirs) + case "move": + deleteEmptySrcDirs, err := in.GetBool("deleteEmptySrcDirs") + if rc.NotErrParamNotFound(err) { + return nil, err + } + return nil, MoveDir(ctx, dstFs, srcFs, deleteEmptySrcDirs, createEmptySrcDirs) + } + panic("unknown rcSyncCopyMove type") +} diff --git a/fs/sync/rc_test.go b/fs/sync/rc_test.go new file mode 100644 index 0000000..a6b66e7 --- /dev/null +++ b/fs/sync/rc_test.go @@ -0,0 +1,97 @@ +package sync + +import ( + "context" + "testing" + + _ "github.com/mewsen/rclone-studip-backend-oot/backend/studip" + "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fs/rc" + "github.com/rclone/rclone/fstest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func rcNewRun(t *testing.T, method string) (*fstest.Run, *rc.Call) { + if *fstest.RemoteName != "" { + t.Skip("Skipping test on non local remote") + } + r := fstest.NewRun(t) + call := rc.Calls.Get(method) + assert.NotNil(t, call) + cache.Put(r.LocalName, r.Flocal) + cache.Put(r.FremoteName, r.Fremote) + return r, call +} + +// sync/copy: copy a directory from source remote to destination remote +func TestRcCopy(t *testing.T) { + r, call := rcNewRun(t, "sync/copy") + r.Mkdir(context.Background(), r.Fremote) + + file1 := r.WriteBoth(context.Background(), "file1", "file1 contents", t1) + file2 := r.WriteFile("subdir/file2", "file2 contents", t2) + file3 := r.WriteObject(context.Background(), "subdir/subsubdir/file3", "file3 contents", t3) + + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file1, file3) + + in := rc.Params{ + "srcFs": r.LocalName, + "dstFs": r.FremoteName, + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file1, file2, file3) +} + +// sync/move: move a directory from source remote to destination remote +func TestRcMove(t *testing.T) { + r, call := rcNewRun(t, "sync/move") + r.Mkdir(context.Background(), r.Fremote) + + file1 := r.WriteBoth(context.Background(), "file1", "file1 contents", t1) + file2 := r.WriteFile("subdir/file2", "file2 contents", t2) + file3 := r.WriteObject(context.Background(), "subdir/subsubdir/file3", "file3 contents", t3) + + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file1, file3) + + in := rc.Params{ + "srcFs": r.LocalName, + "dstFs": r.FremoteName, + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + r.CheckLocalItems(t) + r.CheckRemoteItems(t, file1, file2, file3) +} + +// sync/sync: sync a directory from source remote to destination remote +func TestRcSync(t *testing.T) { + r, call := rcNewRun(t, "sync/sync") + r.Mkdir(context.Background(), r.Fremote) + + file1 := r.WriteBoth(context.Background(), "file1", "file1 contents", t1) + file2 := r.WriteFile("subdir/file2", "file2 contents", t2) + file3 := r.WriteObject(context.Background(), "subdir/subsubdir/file3", "file3 contents", t3) + + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file1, file3) + + in := rc.Params{ + "srcFs": r.LocalName, + "dstFs": r.FremoteName, + } + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file1, file2) +} diff --git a/fs/sync/sync.go b/fs/sync/sync.go new file mode 100644 index 0000000..a425345 --- /dev/null +++ b/fs/sync/sync.go @@ -0,0 +1,1423 @@ +// Package sync is the implementation of sync/copy/move +package sync + +import ( + "context" + "errors" + "fmt" + "path" + "slices" + "sort" + "strings" + "sync" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/march" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/lib/errcount" + "github.com/rclone/rclone/lib/transform" + "golang.org/x/sync/errgroup" +) + +// ErrorMaxDurationReached defines error when transfer duration is reached +// Used for checking on exit and matching to correct exit code. +var ErrorMaxDurationReached = errors.New("max transfer duration reached as set by --max-duration") + +// ErrorMaxDurationReachedFatal is returned from when the max +// duration limit is reached. +var ErrorMaxDurationReachedFatal = fserrors.FatalError(ErrorMaxDurationReached) + +type syncCopyMove struct { + // parameters + fdst fs.Fs + fsrc fs.Fs + deleteMode fs.DeleteMode // how we are doing deletions + DoMove bool + copyEmptySrcDirs bool + deleteEmptySrcDirs bool + dir string + // internal state + ci *fs.ConfigInfo // global config + fi *filter.Filter // filter config + ctx context.Context // internal context for controlling go-routines + cancel func() // cancel the context + inCtx context.Context // internal context for controlling march + inCancel func() // cancel the march context + noTraverse bool // if set don't traverse the dst + noCheckDest bool // if set transfer all objects regardless without checking dst + noUnicodeNormalization bool // don't normalize unicode characters in filenames + deletersWg sync.WaitGroup // for delete before go routine + deleteFilesCh chan fs.Object // channel to receive deletes if delete before + trackRenames bool // set if we should do server-side renames + trackRenamesStrategy trackRenamesStrategy // strategies used for tracking renames + dstFilesMu sync.Mutex // protect dstFiles + dstFiles map[string]fs.Object // dst files, always filled + srcFiles map[string]fs.Object // src files, only used if deleteBefore + srcFilesChan chan fs.Object // passes src objects + srcFilesResult chan error // error result of src listing + dstFilesResult chan error // error result of dst listing + dstEmptyDirsMu sync.Mutex // protect dstEmptyDirs + dstEmptyDirs map[string]fs.DirEntry // potentially empty directories + srcEmptyDirsMu sync.Mutex // protect srcEmptyDirs + srcEmptyDirs map[string]fs.DirEntry // potentially empty directories + srcMoveEmptyDirs map[string]fs.DirEntry // potentially empty directories when moving files out of them + checkerWg sync.WaitGroup // wait for checkers + toBeChecked *pipe // checkers channel + transfersWg sync.WaitGroup // wait for transfers + toBeUploaded *pipe // copiers channel + errorMu sync.Mutex // Mutex covering the errors variables + err error // normal error from copy process + noRetryErr error // error with NoRetry set + fatalErr error // fatal error + commonHash hash.Type // common hash type between src and dst + modifyWindow time.Duration // modify window between fsrc, fdst + renameMapMu sync.Mutex // mutex to protect the below + renameMap map[string][]fs.Object // dst files by hash - only used by trackRenames + renamerWg sync.WaitGroup // wait for renamers + toBeRenamed *pipe // renamers channel + trackRenamesWg sync.WaitGroup // wg for background track renames + trackRenamesCh chan fs.Object // objects are pumped in here + renameCheck []fs.Object // accumulate files to check for rename here + compareCopyDest []fs.Fs // place to check for files to server side copy + backupDir fs.Fs // place to store overwrites/deletes + checkFirst bool // if set run all the checkers before starting transfers + maxDurationEndTime time.Time // end time if --max-duration is set + logger operations.LoggerFn // LoggerFn used to report the results of a sync (or bisync) to an io.Writer + usingLogger bool // whether we are using logger + setDirMetadata bool // if set we set the directory metadata + setDirModTime bool // if set we set the directory modtimes + setDirModTimeAfter bool // if set we set the directory modtimes at the end of the sync + setDirModTimeMu sync.Mutex // protect setDirModTimes and modifiedDirs + setDirModTimes []setDirModTime // directories that need their modtime set + setDirModTimesMaxLevel int // max level of the directories to set + modifiedDirs map[string]struct{} // dirs with changed contents (if s.setDirModTimeAfter) + allowOverlap bool // whether we allow src and dst to overlap (i.e. for convmv) +} + +// For keeping track of delayed modtime sets +type setDirModTime struct { + src fs.Directory + dst fs.Directory + dir string + modTime time.Time + level int // the level of the directory, 0 is root +} + +type trackRenamesStrategy byte + +const ( + trackRenamesStrategyHash trackRenamesStrategy = 1 << iota + trackRenamesStrategyModtime + trackRenamesStrategyLeaf +) + +func (strategy trackRenamesStrategy) hash() bool { + return (strategy & trackRenamesStrategyHash) != 0 +} + +func (strategy trackRenamesStrategy) modTime() bool { + return (strategy & trackRenamesStrategyModtime) != 0 +} + +func (strategy trackRenamesStrategy) leaf() bool { + return (strategy & trackRenamesStrategyLeaf) != 0 +} + +func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool, allowOverlap bool) (*syncCopyMove, error) { + if (deleteMode != fs.DeleteModeOff || DoMove) && operations.OverlappingFilterCheck(ctx, fdst, fsrc) && !allowOverlap { + return nil, fserrors.FatalError(fs.ErrorOverlapping) + } + ci := fs.GetConfig(ctx) + fi := filter.GetConfig(ctx) + s := &syncCopyMove{ + ci: ci, + fi: fi, + fdst: fdst, + fsrc: fsrc, + deleteMode: deleteMode, + DoMove: DoMove, + copyEmptySrcDirs: copyEmptySrcDirs, + deleteEmptySrcDirs: deleteEmptySrcDirs, + dir: "", + srcFilesChan: make(chan fs.Object, ci.Checkers+ci.Transfers), + srcFilesResult: make(chan error, 1), + dstFilesResult: make(chan error, 1), + dstEmptyDirs: make(map[string]fs.DirEntry), + srcEmptyDirs: make(map[string]fs.DirEntry), + srcMoveEmptyDirs: make(map[string]fs.DirEntry), + noTraverse: ci.NoTraverse, + noCheckDest: ci.NoCheckDest, + noUnicodeNormalization: ci.NoUnicodeNormalization, + deleteFilesCh: make(chan fs.Object, ci.Checkers), + trackRenames: ci.TrackRenames, + commonHash: fsrc.Hashes().Overlap(fdst.Hashes()).GetOne(), + modifyWindow: fs.GetModifyWindow(ctx, fsrc, fdst), + trackRenamesCh: make(chan fs.Object, ci.Checkers), + checkFirst: ci.CheckFirst, + setDirMetadata: ci.Metadata && fsrc.Features().ReadDirMetadata && fdst.Features().WriteDirMetadata, + setDirModTime: (!ci.NoUpdateDirModTime && fsrc.Features().CanHaveEmptyDirectories) && (fdst.Features().WriteDirSetModTime || fdst.Features().MkdirMetadata != nil || fdst.Features().DirSetModTime != nil), + setDirModTimeAfter: !ci.NoUpdateDirModTime && (!copyEmptySrcDirs || fsrc.Features().CanHaveEmptyDirectories && fdst.Features().DirModTimeUpdatesOnWrite), + modifiedDirs: make(map[string]struct{}), + allowOverlap: allowOverlap, + } + + s.logger, s.usingLogger = operations.GetLogger(ctx) + + if deleteMode == fs.DeleteModeOff { + loggerOpt := operations.GetLoggerOpt(ctx) + loggerOpt.DeleteModeOff = true + loggerOpt.LoggerFn = s.logger + ctx = operations.WithLoggerOpt(ctx, loggerOpt) + } + + backlog := ci.MaxBacklog + if s.checkFirst { + fs.Infof(s.fdst, "Running all checks before starting transfers") + backlog = -1 + } + var err error + s.toBeChecked, err = newPipe(ci.OrderBy, accounting.Stats(ctx).SetCheckQueue, backlog) + if err != nil { + return nil, err + } + s.toBeUploaded, err = newPipe(ci.OrderBy, accounting.Stats(ctx).SetTransferQueue, backlog) + if err != nil { + return nil, err + } + s.toBeRenamed, err = newPipe(ci.OrderBy, accounting.Stats(ctx).SetRenameQueue, backlog) + if err != nil { + return nil, err + } + if ci.MaxDuration > 0 { + s.maxDurationEndTime = time.Now().Add(time.Duration(ci.MaxDuration)) + fs.Infof(s.fdst, "Transfer session %v deadline: %s", ci.CutoffMode, s.maxDurationEndTime.Format("2006/01/02 15:04:05")) + } + // If a max session duration has been defined add a deadline + // to the main context if cutoff mode is hard. This will cut + // the transfers off. + if !s.maxDurationEndTime.IsZero() && ci.CutoffMode == fs.CutoffModeHard { + s.ctx, s.cancel = context.WithDeadline(ctx, s.maxDurationEndTime) + } else { + s.ctx, s.cancel = context.WithCancel(ctx) + } + // Input context - cancel this for graceful stop. + // + // If a max session duration has been defined add a deadline + // to the input context if cutoff mode is graceful or soft. + // This won't stop the transfers but will cut the + // list/check/transfer pipelines. + if !s.maxDurationEndTime.IsZero() && ci.CutoffMode != fs.CutoffModeHard { + s.inCtx, s.inCancel = context.WithDeadline(s.ctx, s.maxDurationEndTime) + } else { + s.inCtx, s.inCancel = context.WithCancel(s.ctx) + } + if s.noTraverse && s.deleteMode != fs.DeleteModeOff { + if !fi.HaveFilesFrom() { + fs.Errorf(nil, "Ignoring --no-traverse with sync") + } + s.noTraverse = false + } + s.trackRenamesStrategy, err = parseTrackRenamesStrategy(ci.TrackRenamesStrategy) + if err != nil { + return nil, err + } + if s.noCheckDest { + if s.deleteMode != fs.DeleteModeOff { + return nil, errors.New("can't use --no-check-dest with sync: use copy instead") + } + if ci.Immutable { + return nil, errors.New("can't use --no-check-dest with --immutable") + } + if s.backupDir != nil { + return nil, errors.New("can't use --no-check-dest with --backup-dir") + } + } + if s.trackRenames { + // Don't track renames for remotes without server-side move support. + if !operations.CanServerSideMove(fdst) { + fs.Errorf(fdst, "Ignoring --track-renames as the destination does not support server-side move or copy") + s.trackRenames = false + } + if s.trackRenamesStrategy.hash() && s.commonHash == hash.None { + fs.Errorf(fdst, "Ignoring --track-renames as the source and destination do not have a common hash") + s.trackRenames = false + } + + if s.trackRenamesStrategy.modTime() && s.modifyWindow == fs.ModTimeNotSupported { + fs.Errorf(fdst, "Ignoring --track-renames as either the source or destination do not support modtime") + s.trackRenames = false + } + + if s.deleteMode == fs.DeleteModeOff { + fs.Errorf(fdst, "Ignoring --track-renames as it doesn't work with copy or move, only sync") + s.trackRenames = false + } + } + if s.trackRenames { + // track renames needs delete after + if s.deleteMode != fs.DeleteModeOff { + s.deleteMode = fs.DeleteModeAfter + } + if s.noTraverse { + fs.Errorf(nil, "Ignoring --no-traverse with --track-renames") + s.noTraverse = false + } + } + // Make Fs for --backup-dir if required + if ci.BackupDir != "" || ci.Suffix != "" { + var err error + s.backupDir, err = operations.BackupDir(ctx, fdst, fsrc, "") + if err != nil { + return nil, err + } + } + if len(ci.CompareDest) > 0 { + var err error + s.compareCopyDest, err = operations.GetCompareDest(ctx) + if err != nil { + return nil, err + } + } else if len(ci.CopyDest) > 0 { + var err error + s.compareCopyDest, err = operations.GetCopyDest(ctx, fdst) + if err != nil { + return nil, err + } + } + return s, nil +} + +// Check to see if the context has been cancelled +func (s *syncCopyMove) aborting() bool { + return s.ctx.Err() != nil +} + +// This reads the map and pumps it into the channel passed in, closing +// the channel at the end +func (s *syncCopyMove) pumpMapToChan(files map[string]fs.Object, out chan<- fs.Object) { +outer: + for _, o := range files { + if s.aborting() { + break outer + } + select { + case out <- o: + case <-s.ctx.Done(): + break outer + } + } + close(out) + s.srcFilesResult <- nil +} + +// This checks the types of errors returned while copying files +func (s *syncCopyMove) processError(err error) { + if err == nil { + return + } + if err == context.DeadlineExceeded { + err = fserrors.NoRetryError(err) + } else if err == accounting.ErrorMaxTransferLimitReachedGraceful { + if s.inCtx.Err() == nil { + fs.Logf(nil, "%v - stopping transfers", err) + // Cancel the march and stop the pipes + s.inCancel() + } + } else if err == context.Canceled && s.inCtx.Err() != nil { + // Ignore context Canceled if we have called s.inCancel() + return + } + s.errorMu.Lock() + defer s.errorMu.Unlock() + switch { + case fserrors.IsFatalError(err): + if !s.aborting() { + fs.Errorf(nil, "Cancelling sync due to fatal error: %v", err) + s.cancel() + } + s.fatalErr = err + case fserrors.IsNoRetryError(err): + s.noRetryErr = err + default: + s.err = err + } +} + +// Returns the current error (if any) in the order of precedence +// +// fatalErr +// normal error +// noRetryErr +func (s *syncCopyMove) currentError() error { + s.errorMu.Lock() + defer s.errorMu.Unlock() + if s.fatalErr != nil { + return s.fatalErr + } + if s.err != nil { + return s.err + } + return s.noRetryErr +} + +// pairChecker reads Objects~s on in send to out if they need transferring. +// +// FIXME potentially doing lots of hashes at once +func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.WaitGroup) { + defer wg.Done() + for { + pair, ok := in.GetMax(s.inCtx, fraction) + if !ok { + return + } + src := pair.Src + var err error + tr := accounting.Stats(s.ctx).NewCheckingTransfer(src, "checking") + // Check to see if can store this + if src.Storable() { + needTransfer := operations.NeedTransfer(s.ctx, pair.Dst, pair.Src) + if needTransfer { + NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, pair.Dst, pair.Src, s.compareCopyDest, s.backupDir) + if err != nil { + s.processError(err) + s.logger(s.ctx, operations.TransferError, pair.Src, pair.Dst, err) + } + if NoNeedTransfer { + needTransfer = false + } + } + // Fix case for case insensitive filesystems + if s.ci.FixCase && !s.ci.Immutable && src.Remote() != pair.Dst.Remote() { + if newDst, err := operations.Move(s.ctx, s.fdst, nil, src.Remote(), pair.Dst); err != nil { + fs.Errorf(pair.Dst, "Error while attempting to rename to %s: %v", src.Remote(), err) + s.processError(err) + } else { + fs.Infof(pair.Dst, "Fixed case by renaming to: %s", src.Remote()) + pair.Dst = newDst + } + } + if needTransfer { + // If files are treated as immutable, fail if destination exists and does not match + if s.ci.Immutable && pair.Dst != nil { + err := fs.CountError(s.ctx, fserrors.NoRetryError(fs.ErrorImmutableModified)) + fs.Errorf(pair.Dst, "Source and destination exist but do not match: %v", err) + s.processError(err) + } else { + if pair.Dst != nil { + s.markDirModifiedObject(pair.Dst) + } else { + s.markDirModifiedObject(src) + } + // If destination already exists, then we must move it into --backup-dir if required + if pair.Dst != nil && s.backupDir != nil { + err := operations.MoveBackupDir(s.ctx, s.backupDir, pair.Dst) + if err != nil { + s.processError(err) + s.logger(s.ctx, operations.TransferError, pair.Src, pair.Dst, err) + } else { + // If successful zero out the dst as it is no longer there and copy the file + pair.Dst = nil + ok = out.Put(s.inCtx, pair) + if !ok { + return + } + } + } else { + ok = out.Put(s.inCtx, pair) + if !ok { + return + } + } + } + } else { + // If moving need to delete the files we don't need to copy + if s.DoMove { + // Delete src if no error on copy + if operations.SameObject(src, pair.Dst) { + fs.Logf(src, "Not removing source file as it is the same file as the destination") + } else if s.ci.IgnoreExisting { + fs.Debugf(src, "Not removing source file as destination file exists and --ignore-existing is set") + } else if s.checkFirst && s.ci.OrderBy != "" { + // If we want perfect ordering then use the transfers to delete the file + // + // We send src == dst, to say we want the src deleted + ok = out.Put(s.inCtx, fs.ObjectPair{Src: src, Dst: src}) + if !ok { + return + } + } else { + deleteFileErr := operations.DeleteFile(s.ctx, src) + s.processError(deleteFileErr) + s.logger(s.ctx, operations.TransferError, pair.Src, pair.Dst, deleteFileErr) + } + } + } + } + tr.Done(s.ctx, err) + } +} + +// pairRenamer reads Objects~s on in and attempts to rename them, +// otherwise it sends them out if they need transferring. +func (s *syncCopyMove) pairRenamer(in *pipe, out *pipe, fraction int, wg *sync.WaitGroup) { + defer wg.Done() + for { + pair, ok := in.GetMax(s.inCtx, fraction) + if !ok { + return + } + src := pair.Src + if !s.tryRename(src) { + // pass on if not renamed + fs.Debugf(src, "Need to transfer - No matching file found at Destination") + ok = out.Put(s.inCtx, pair) + if !ok { + return + } + } + } +} + +// pairCopyOrMove reads Objects on in and moves or copies them. +func (s *syncCopyMove) pairCopyOrMove(ctx context.Context, in *pipe, fdst fs.Fs, fraction int, wg *sync.WaitGroup) { + defer wg.Done() + var err error + for { + pair, ok := in.GetMax(s.inCtx, fraction) + if !ok { + return + } + src := pair.Src + dst := pair.Dst + if s.DoMove { + if src != dst { + _, err = operations.MoveTransfer(ctx, fdst, dst, src.Remote(), src) + } else { + // src == dst signals delete the src + err = operations.DeleteFile(ctx, src) + } + } else { + _, err = operations.Copy(ctx, fdst, dst, src.Remote(), src) + } + s.processError(err) + if err != nil { + s.logger(ctx, operations.TransferError, src, dst, err) + } + } +} + +// This starts the background checkers. +func (s *syncCopyMove) startCheckers() { + s.checkerWg.Add(s.ci.Checkers) + for i := range s.ci.Checkers { + fraction := (100 * i) / s.ci.Checkers + go s.pairChecker(s.toBeChecked, s.toBeUploaded, fraction, &s.checkerWg) + } +} + +// This stops the background checkers +func (s *syncCopyMove) stopCheckers() { + s.toBeChecked.Close() + fs.Debugf(s.fdst, "Waiting for checks to finish") + s.checkerWg.Wait() +} + +// This starts the background transfers +func (s *syncCopyMove) startTransfers() { + s.transfersWg.Add(s.ci.Transfers) + for i := range s.ci.Transfers { + fraction := (100 * i) / s.ci.Transfers + go s.pairCopyOrMove(s.ctx, s.toBeUploaded, s.fdst, fraction, &s.transfersWg) + } +} + +// This stops the background transfers +func (s *syncCopyMove) stopTransfers() { + s.toBeUploaded.Close() + fs.Debugf(s.fdst, "Waiting for transfers to finish") + s.transfersWg.Wait() +} + +// This starts the background renamers. +func (s *syncCopyMove) startRenamers() { + if !s.trackRenames { + return + } + s.renamerWg.Add(s.ci.Checkers) + for i := range s.ci.Checkers { + fraction := (100 * i) / s.ci.Checkers + go s.pairRenamer(s.toBeRenamed, s.toBeUploaded, fraction, &s.renamerWg) + } +} + +// This stops the background renamers +func (s *syncCopyMove) stopRenamers() { + if !s.trackRenames { + return + } + s.toBeRenamed.Close() + fs.Debugf(s.fdst, "Waiting for renames to finish") + s.renamerWg.Wait() +} + +// This starts the collection of possible renames +func (s *syncCopyMove) startTrackRenames() { + if !s.trackRenames { + return + } + s.trackRenamesWg.Add(1) + go func() { + defer s.trackRenamesWg.Done() + for o := range s.trackRenamesCh { + s.renameCheck = append(s.renameCheck, o) + } + }() +} + +// This stops the background rename collection +func (s *syncCopyMove) stopTrackRenames() { + if !s.trackRenames { + return + } + close(s.trackRenamesCh) + s.trackRenamesWg.Wait() +} + +// This starts the background deletion of files for --delete-during +func (s *syncCopyMove) startDeleters() { + if s.deleteMode != fs.DeleteModeDuring && s.deleteMode != fs.DeleteModeOnly { + return + } + s.deletersWg.Add(1) + go func() { + defer s.deletersWg.Done() + err := operations.DeleteFilesWithBackupDir(s.ctx, s.deleteFilesCh, s.backupDir) + s.processError(err) + }() +} + +// This stops the background deleters +func (s *syncCopyMove) stopDeleters() { + if s.deleteMode != fs.DeleteModeDuring && s.deleteMode != fs.DeleteModeOnly { + return + } + close(s.deleteFilesCh) + s.deletersWg.Wait() +} + +// This deletes the files in the dstFiles map. If checkSrcMap is set +// then it checks to see if they exist first in srcFiles the source +// file map, otherwise it unconditionally deletes them. If +// checkSrcMap is clear then it assumes that the any source files that +// have been found have been removed from dstFiles already. +func (s *syncCopyMove) deleteFiles(checkSrcMap bool) error { + if accounting.Stats(s.ctx).Errored() && !s.ci.IgnoreErrors { + fs.Errorf(s.fdst, "%v", fs.ErrorNotDeleting) + // log all deletes as errors + for remote, o := range s.dstFiles { + if checkSrcMap { + _, exists := s.srcFiles[remote] + if exists { + continue + } + } + s.logger(s.ctx, operations.TransferError, nil, o, fs.ErrorNotDeleting) + } + return fs.ErrorNotDeleting + } + + // Delete the spare files + toDelete := make(fs.ObjectsChan, s.ci.Checkers) + go func() { + outer: + for remote, o := range s.dstFiles { + if checkSrcMap { + _, exists := s.srcFiles[remote] + if exists { + continue + } + } + if s.aborting() { + break + } + select { + case <-s.ctx.Done(): + break outer + case toDelete <- o: + } + } + close(toDelete) + }() + return operations.DeleteFilesWithBackupDir(s.ctx, toDelete, s.backupDir) +} + +// This deletes the empty directories in the slice passed in. It +// ignores any errors deleting directories +func (s *syncCopyMove) deleteEmptyDirectories(ctx context.Context, f fs.Fs, entriesMap map[string]fs.DirEntry) error { + if len(entriesMap) == 0 { + return nil + } + if accounting.Stats(ctx).Errored() && !s.ci.IgnoreErrors { + fs.Errorf(f, "%v", fs.ErrorNotDeletingDirs) + return fs.ErrorNotDeletingDirs + } + + var entries fs.DirEntries + for _, entry := range entriesMap { + entries = append(entries, entry) + } + // Now delete the empty directories starting from the longest path + sort.Sort(entries) + var errorCount int + var okCount int + for i := len(entries) - 1; i >= 0; i-- { + entry := entries[i] + dir, ok := entry.(fs.Directory) + if ok { + // TryRmdir only deletes empty directories + err := operations.TryRmdir(ctx, f, dir.Remote()) + if err != nil { + fs.Debugf(fs.LogDirName(f, dir.Remote()), "Failed to Rmdir: %v", err) + errorCount++ + } else { + okCount++ + } + } else { + fs.Errorf(f, "Not a directory: %v", entry) + } + } + if errorCount > 0 { + fs.Debugf(f, "failed to delete %d directories", errorCount) + } + if okCount > 0 { + fs.Debugf(f, "deleted %d directories", okCount) + } + return nil +} + +// mark the parent of entry as not empty and if entry is a directory mark it as potentially empty. +func (s *syncCopyMove) markParentNotEmpty(entry fs.DirEntry) { + s.srcEmptyDirsMu.Lock() + defer s.srcEmptyDirsMu.Unlock() + // Mark entry as potentially empty if it is a directory + _, isDir := entry.(fs.Directory) + if isDir { + s.srcEmptyDirs[entry.Remote()] = entry + // if DoMove and --delete-empty-src-dirs flag is set then record the parent but + // don't remove any as we are about to move files out of them them making the + // directory empty. + if s.DoMove && s.deleteEmptySrcDirs { + s.srcMoveEmptyDirs[entry.Remote()] = entry + } + } + parentDir := path.Dir(entry.Remote()) + if isDir && s.copyEmptySrcDirs { + // Mark its parent as not empty + if parentDir == "." { + parentDir = "" + } + delete(s.srcEmptyDirs, parentDir) + } + if !isDir { + // Mark ALL its parents as not empty + for { + if parentDir == "." { + parentDir = "" + } + delete(s.srcEmptyDirs, parentDir) + if parentDir == "" || parentDir == "/" { + break + } + parentDir = path.Dir(parentDir) + } + } +} + +// parseTrackRenamesStrategy turns a config string into a trackRenamesStrategy +func parseTrackRenamesStrategy(strategies string) (strategy trackRenamesStrategy, err error) { + if len(strategies) == 0 { + return strategy, nil + } + for s := range strings.SplitSeq(strategies, ",") { + switch s { + case "hash": + strategy |= trackRenamesStrategyHash + case "modtime": + strategy |= trackRenamesStrategyModtime + case "leaf": + strategy |= trackRenamesStrategyLeaf + case "size": + // ignore + default: + return strategy, fmt.Errorf("unknown track renames strategy %q", s) + } + } + return strategy, nil +} + +// renameID makes a string with the size and the other identifiers of the requested rename strategies +// +// it may return an empty string in which case no hash could be made +func (s *syncCopyMove) renameID(obj fs.Object, renamesStrategy trackRenamesStrategy, precision time.Duration) string { + var builder strings.Builder + + fmt.Fprintf(&builder, "%d", obj.Size()) + + if renamesStrategy.hash() { + var err error + hash, err := obj.Hash(s.ctx, s.commonHash) + if err != nil { + fs.Debugf(obj, "Hash failed: %v", err) + return "" + } + if hash == "" { + return "" + } + + builder.WriteRune(',') + builder.WriteString(hash) + } + + // for renamesStrategy.modTime() we don't add to the hash but we check the times in + // popRenameMap + + if renamesStrategy.leaf() { + builder.WriteRune(',') + builder.WriteString(path.Base(obj.Remote())) + } + + return builder.String() +} + +// pushRenameMap adds the object with hash to the rename map +func (s *syncCopyMove) pushRenameMap(hash string, obj fs.Object) { + s.renameMapMu.Lock() + s.renameMap[hash] = append(s.renameMap[hash], obj) + s.renameMapMu.Unlock() +} + +// popRenameMap finds the object with hash and pop the first match from +// renameMap or returns nil if not found. +func (s *syncCopyMove) popRenameMap(hash string, src fs.Object) (dst fs.Object) { + s.renameMapMu.Lock() + defer s.renameMapMu.Unlock() + dsts, ok := s.renameMap[hash] + if ok && len(dsts) > 0 { + // Element to remove + i := 0 + + // If using track renames strategy modtime then we need to check the modtimes here + if s.trackRenamesStrategy.modTime() { + i = -1 + srcModTime := src.ModTime(s.ctx) + for j, dst := range dsts { + dstModTime := dst.ModTime(s.ctx) + dt := dstModTime.Sub(srcModTime) + if dt < s.modifyWindow && dt > -s.modifyWindow { + i = j + break + } + } + // If nothing matched then return nil + if i < 0 { + return nil + } + } + + // Remove the entry and return it + dst = dsts[i] + dsts = slices.Delete(dsts, i, i+1) + if len(dsts) > 0 { + s.renameMap[hash] = dsts + } else { + delete(s.renameMap, hash) + } + } + return dst +} + +// makeRenameMap builds a map of the destination files by hash that +// match sizes in the slice of objects in s.renameCheck +func (s *syncCopyMove) makeRenameMap() { + fs.Infof(s.fdst, "Making map for --track-renames") + + // first make a map of possible sizes we need to check + possibleSizes := map[int64]struct{}{} + for _, obj := range s.renameCheck { + possibleSizes[obj.Size()] = struct{}{} + } + + // pump all the dstFiles into in + in := make(chan fs.Object, s.ci.Checkers) + go s.pumpMapToChan(s.dstFiles, in) + + // now make a map of size,hash for all dstFiles + s.renameMap = make(map[string][]fs.Object) + var wg sync.WaitGroup + wg.Add(s.ci.Checkers) + for range s.ci.Checkers { + go func() { + defer wg.Done() + for obj := range in { + // only create hash for dst fs.Object if its size could match + if _, found := possibleSizes[obj.Size()]; found { + tr := accounting.Stats(s.ctx).NewCheckingTransfer(obj, "renaming") + hash := s.renameID(obj, s.trackRenamesStrategy, s.modifyWindow) + + if hash != "" { + s.pushRenameMap(hash, obj) + } + + tr.Done(s.ctx, nil) + } + } + }() + } + wg.Wait() + fs.Infof(s.fdst, "Finished making map for --track-renames") +} + +// tryRename renames an src object when doing track renames if +// possible, it returns true if the object was renamed. +func (s *syncCopyMove) tryRename(src fs.Object) bool { + // Calculate the hash of the src object + hash := s.renameID(src, s.trackRenamesStrategy, fs.GetModifyWindow(s.ctx, s.fsrc, s.fdst)) + + if hash == "" { + return false + } + + // Get a match on fdst + dst := s.popRenameMap(hash, src) + if dst == nil { + return false + } + + // Find dst object we are about to overwrite if it exists + dstOverwritten, _ := s.fdst.NewObject(s.ctx, src.Remote()) + + // Rename dst to have name src.Remote() + _, err := operations.Move(s.ctx, s.fdst, dstOverwritten, src.Remote(), dst) + if err != nil { + fs.Debugf(src, "Failed to rename to %q: %v", dst.Remote(), err) + return false + } + + // remove file from dstFiles if present + s.dstFilesMu.Lock() + delete(s.dstFiles, dst.Remote()) + s.dstFilesMu.Unlock() + + fs.Infof(src, "Renamed from %q", dst.Remote()) + return true +} + +// Syncs fsrc into fdst +// +// If Delete is true then it deletes any files in fdst that aren't in fsrc +// +// If DoMove is true then files will be moved instead of copied. +// +// dir is the start directory, "" for root +func (s *syncCopyMove) run() error { + if operations.Same(s.fdst, s.fsrc) && !s.allowOverlap { + fs.Errorf(s.fdst, "Nothing to do as source and destination are the same") + return nil + } + + // Start background checking and transferring pipeline + s.startCheckers() + s.startRenamers() + if !s.checkFirst { + s.startTransfers() + } + s.startDeleters() + s.dstFiles = make(map[string]fs.Object) + + s.startTrackRenames() + + // set up a march over fdst and fsrc + m := &march.March{ + Ctx: s.inCtx, + Fdst: s.fdst, + Fsrc: s.fsrc, + Dir: s.dir, + NoTraverse: s.noTraverse, + Callback: s, + DstIncludeAll: s.fi.Opt.DeleteExcluded, + NoCheckDest: s.noCheckDest, + NoUnicodeNormalization: s.noUnicodeNormalization, + } + s.processError(m.Run(s.ctx)) + + s.stopTrackRenames() + if s.trackRenames { + // Build the map of the remaining dstFiles by hash + s.makeRenameMap() + // Attempt renames for all the files which don't have a matching dst + for _, src := range s.renameCheck { + ok := s.toBeRenamed.Put(s.inCtx, fs.ObjectPair{Src: src, Dst: nil}) + if !ok { + break + } + } + } + + // Stop background checking and transferring pipeline + s.stopCheckers() + if s.checkFirst { + fs.Infof(s.fdst, "Checks finished, now starting transfers") + s.startTransfers() + } + s.stopRenamers() + s.stopTransfers() + s.stopDeleters() + + // Delete files after + if s.deleteMode == fs.DeleteModeAfter { + if s.currentError() != nil && !s.ci.IgnoreErrors { + fs.Errorf(s.fdst, "%v", fs.ErrorNotDeleting) + } else { + s.processError(s.deleteFiles(false)) + } + } + + // Update modtimes for directories if necessary + if s.setDirModTime && s.setDirModTimeAfter { + s.processError(s.setDelayedDirModTimes(s.ctx)) + } + + // Prune empty directories + if s.deleteMode != fs.DeleteModeOff { + if s.currentError() != nil && !s.ci.IgnoreErrors { + fs.Errorf(s.fdst, "%v", fs.ErrorNotDeletingDirs) + } else { + s.processError(s.deleteEmptyDirectories(s.ctx, s.fdst, s.dstEmptyDirs)) + } + } + + // Delete empty fsrc subdirectories + // if DoMove and --delete-empty-src-dirs flag is set + if s.DoMove && s.deleteEmptySrcDirs { + // delete potentially empty subdirectories that were part of the move + s.processError(s.deleteEmptyDirectories(s.ctx, s.fsrc, s.srcMoveEmptyDirs)) + } + + // Read the error out of the contexts if there is one + s.processError(s.ctx.Err()) + s.processError(s.inCtx.Err()) + + // If the duration was exceeded then add a Fatal Error so we don't retry + if !s.maxDurationEndTime.IsZero() && time.Since(s.maxDurationEndTime) > 0 { + fs.Errorf(s.fdst, "%v", ErrorMaxDurationReachedFatal) + s.processError(ErrorMaxDurationReachedFatal) + } + + // Print nothing to transfer message if there were no transfers and no errors + if s.deleteMode != fs.DeleteModeOnly && accounting.Stats(s.ctx).GetTransfers() == 0 && s.currentError() == nil { + fs.Infof(nil, "There was nothing to transfer") + } + + // cancel the contexts to free resources + s.inCancel() + s.cancel() + return s.currentError() +} + +// DstOnly have an object which is in the destination only +func (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) { + if s.deleteMode == fs.DeleteModeOff { + if s.usingLogger { + switch x := dst.(type) { + case fs.Object: + s.logger(s.ctx, operations.MissingOnSrc, nil, x, nil) + case fs.Directory: + // it's a directory that we'd normally skip, because we're not deleting anything on the dest + // however, to make sure every file is logged, we need to list it, so we need to return true here. + // we skip this when not using logger. + s.logger(s.ctx, operations.MissingOnSrc, nil, dst, fs.ErrorIsDir) + return true + } + } + return false + } + switch x := dst.(type) { + case fs.Object: + s.logger(s.ctx, operations.MissingOnSrc, nil, x, nil) + switch s.deleteMode { + case fs.DeleteModeAfter: + // record object as needs deleting + s.dstFilesMu.Lock() + s.dstFiles[x.Remote()] = x + s.dstFilesMu.Unlock() + case fs.DeleteModeDuring, fs.DeleteModeOnly: + select { + case <-s.ctx.Done(): + return + case s.deleteFilesCh <- x: + } + default: + panic(fmt.Sprintf("unexpected delete mode %d", s.deleteMode)) + } + case fs.Directory: + // Do the same thing to the entire contents of the directory + // Record directory as it is potentially empty and needs deleting + if s.fdst.Features().CanHaveEmptyDirectories { + s.dstEmptyDirsMu.Lock() + s.dstEmptyDirs[dst.Remote()] = dst + s.dstEmptyDirsMu.Unlock() + s.logger(s.ctx, operations.MissingOnSrc, nil, dst, fs.ErrorIsDir) + } + return true + default: + panic("Bad object in DirEntries") + + } + return false +} + +// keeps track of dirs with changed contents, to avoid setting modtimes on dirs that haven't changed +func (s *syncCopyMove) markDirModified(dir string) { + if !s.setDirModTimeAfter { + return + } + s.setDirModTimeMu.Lock() + defer s.setDirModTimeMu.Unlock() + s.modifiedDirs[dir] = struct{}{} +} + +// like markDirModified, but accepts an Object instead of a string. +// the marked dir will be this object's parent. +func (s *syncCopyMove) markDirModifiedObject(o fs.Object) { + dir := path.Dir(o.Remote()) + if dir == "." { + dir = "" + } + s.markDirModified(dir) +} + +// copyDirMetadata copies the src directory modTime or Metadata to dst +// or f if nil. If dst is nil then it uses dir as the name of the new +// directory. +// +// It returns the destination directory if possible. Note that this may +// be nil. +func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory) { + var err error + if dst != nil && src.Remote() == dst.Remote() && operations.OverlappingFilterCheck(ctx, s.fdst, s.fsrc) { + return nil // src and dst can be the same in convmv + } + equal := operations.DirsEqual(ctx, src, dst, operations.DirsEqualOpt{ModifyWindow: s.modifyWindow, SetDirModtime: s.setDirModTime, SetDirMetadata: s.setDirMetadata}) + if !s.setDirModTimeAfter && equal { + return nil + } + newDst = dst + if !equal { + if s.setDirMetadata && s.copyEmptySrcDirs { + newDst, err = operations.CopyDirMetadata(ctx, f, dst, dir, src) + } else if dst == nil && s.setDirModTime && s.copyEmptySrcDirs { + newDst, err = operations.MkdirModTime(ctx, f, dir, src.ModTime(ctx)) + } else if dst == nil && s.copyEmptySrcDirs { + err = operations.Mkdir(ctx, f, dir) + } else if dst != nil && s.setDirModTime { + newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx)) + } + } + if transform.Transforming(ctx) && newDst != nil && src.Remote() != newDst.Remote() { + s.markParentNotEmpty(src) + } + // If we need to set modtime after and we created a dir, then save it for later + if s.setDirModTime && s.setDirModTimeAfter && err == nil { + if newDst != nil { + dir = newDst.Remote() + } + level := strings.Count(dir, "/") + 1 + // The root directory "" is at the top level + if dir == "" { + level = 0 + } + s.setDirModTimeMu.Lock() + // Keep track of the maximum level inserted + if level > s.setDirModTimesMaxLevel { + s.setDirModTimesMaxLevel = level + } + set := setDirModTime{ + src: src, + dst: newDst, + dir: dir, + modTime: src.ModTime(ctx), + level: level, + } + s.setDirModTimes = append(s.setDirModTimes, set) + s.setDirModTimeMu.Unlock() + fs.Debugf(nil, "Added delayed dir = %q, newDst=%v", dir, newDst) + } + s.processError(err) + if err != nil { + return nil + } + return newDst +} + +// Set the modtimes for directories +func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error { + s.setDirModTimeMu.Lock() + defer s.setDirModTimeMu.Unlock() + + // Timestamp all directories at the same level in parallel, deepest first + // We do this by iterating the slice multiple times to save memory + // There could be a lot of directories in this slice. + errCount := errcount.New() + for level := s.setDirModTimesMaxLevel; level >= 0; level-- { + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(s.ci.Checkers) + for _, item := range s.setDirModTimes { + if item.level != level { + continue + } + // End early if error + if gCtx.Err() != nil { + break + } + if _, ok := s.modifiedDirs[item.dir]; !ok { + continue + } + if !s.copyEmptySrcDirs { + if _, isEmpty := s.srcEmptyDirs[item.dir]; isEmpty { + continue + } + } + item := item + if s.setDirModTimeAfter { // mark dir's parent as modified + dir := path.Dir(item.dir) + if dir == "." { + dir = "" + } + s.modifiedDirs[dir] = struct{}{} // lock is already held + } + g.Go(func() error { + var err error + if s.setDirMetadata { + _, err = operations.CopyDirMetadata(gCtx, s.fdst, item.dst, item.dir, item.src) + } else { + _, err = operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime) + } + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(item.dir, "Failed to update directory timestamp or metadata: %v", err) + errCount.Add(err) + } + return nil // don't return errors, just count them + }) + } + err := g.Wait() + if err != nil { + return err + } + } + return errCount.Err("failed to set directory modtime") +} + +// SrcOnly have an object which is in the source only +func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) { + if s.deleteMode == fs.DeleteModeOnly { + return false + } + switch x := src.(type) { + case fs.Object: + s.logger(s.ctx, operations.MissingOnDst, x, nil, nil) + s.markParentNotEmpty(src) + + if s.trackRenames { + // Save object to check for a rename later + select { + case <-s.ctx.Done(): + return + case s.trackRenamesCh <- x: + } + } else { + // Check CompareDest && CopyDest + NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, nil, x, s.compareCopyDest, s.backupDir) + if err != nil { + s.processError(err) + s.logger(s.ctx, operations.TransferError, x, nil, err) + } + if !NoNeedTransfer { + // No need to check since doesn't exist + fs.Debugf(src, "Need to transfer - File not found at Destination") + s.markDirModifiedObject(x) + ok := s.toBeUploaded.Put(s.inCtx, fs.ObjectPair{Src: x, Dst: nil}) + if !ok { + return + } + } + } + case fs.Directory: + // Do the same thing to the entire contents of the directory + s.markParentNotEmpty(src) + s.logger(s.ctx, operations.MissingOnDst, src, nil, fs.ErrorIsDir) + + // Create the directory and make sure the Metadata/ModTime is correct + s.copyDirMetadata(s.ctx, s.fdst, nil, transform.Path(s.ctx, x.Remote(), true), x) + s.markDirModified(transform.Path(s.ctx, x.Remote(), true)) + return true + default: + panic("Bad object in DirEntries") + } + return false +} + +// Match is called when src and dst are present, so sync src to dst +func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse bool) { + switch srcX := src.(type) { + case fs.Object: + s.markParentNotEmpty(src) + + if s.deleteMode == fs.DeleteModeOnly { + return false + } + dstX, ok := dst.(fs.Object) + if ok { + // No logger here because we'll handle it in equal() + ok = s.toBeChecked.Put(s.inCtx, fs.ObjectPair{Src: srcX, Dst: dstX}) + if !ok { + return false + } + } else { + // FIXME src is file, dst is directory + err := errors.New("can't overwrite directory with file") + fs.Errorf(dst, "%v", err) + s.processError(err) + s.logger(ctx, operations.TransferError, srcX, dstX, err) + } + case fs.Directory: + // Do the same thing to the entire contents of the directory + srcX = fs.NewOverrideDirectory(srcX, transform.Path(ctx, src.Remote(), true)) + src = srcX + if !transform.Transforming(ctx) || src.Remote() != dst.Remote() { + s.markParentNotEmpty(src) + } + dstX, ok := dst.(fs.Directory) + if ok { + s.logger(s.ctx, operations.Match, src, dst, fs.ErrorIsDir) + // Create the directory and make sure the Metadata/ModTime is correct + s.copyDirMetadata(s.ctx, s.fdst, dstX, "", srcX) + + if s.ci.FixCase && !s.ci.Immutable && src.Remote() != dst.Remote() { + // Fix case for case insensitive filesystems + // Fix each dir before recursing into subdirs and files + err := operations.DirMoveCaseInsensitive(s.ctx, s.fdst, dst.Remote(), src.Remote()) + if err != nil { + fs.Errorf(dst, "Error while attempting to rename to %s: %v", src.Remote(), err) + s.processError(err) + } else { + fs.Infof(dst, "Fixed case by renaming to: %s", src.Remote()) + } + } + + return true + } + // FIXME src is dir, dst is file + err := errors.New("can't overwrite file with directory") + fs.Errorf(dst, "%v", err) + s.processError(err) + s.logger(ctx, operations.TransferError, src.(fs.ObjectInfo), dst.(fs.ObjectInfo), err) + default: + panic("Bad object in DirEntries") + } + return false +} + +// Syncs fsrc into fdst +// +// If Delete is true then it deletes any files in fdst that aren't in fsrc +// +// If DoMove is true then files will be moved instead of copied. +// +// dir is the start directory, "" for root +func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool, allowOverlap bool) error { + ci := fs.GetConfig(ctx) + if deleteMode != fs.DeleteModeOff && DoMove { + return fserrors.FatalError(errors.New("can't delete and move at the same time")) + } + // Run an extra pass to delete only + if deleteMode == fs.DeleteModeBefore { + if ci.TrackRenames { + return fserrors.FatalError(errors.New("can't use --delete-before with --track-renames")) + } + // only delete stuff during in this pass + do, err := newSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOnly, false, deleteEmptySrcDirs, copyEmptySrcDirs, allowOverlap) + if err != nil { + return err + } + err = do.run() + if err != nil { + return err + } + // Next pass does a copy only + deleteMode = fs.DeleteModeOff + } + do, err := newSyncCopyMove(ctx, fdst, fsrc, deleteMode, DoMove, deleteEmptySrcDirs, copyEmptySrcDirs, allowOverlap) + if err != nil { + return err + } + return do.run() +} + +// Sync fsrc into fdst +func Sync(ctx context.Context, fdst, fsrc fs.Fs, copyEmptySrcDirs bool) error { + ci := fs.GetConfig(ctx) + return runSyncCopyMove(ctx, fdst, fsrc, ci.DeleteMode, false, false, copyEmptySrcDirs, false) +} + +// CopyDir copies fsrc into fdst +func CopyDir(ctx context.Context, fdst, fsrc fs.Fs, copyEmptySrcDirs bool) error { + return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, false, false, copyEmptySrcDirs, false) +} + +// moveDir moves fsrc into fdst +func moveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error { + return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs, false) +} + +// Transform renames fdst in place +func Transform(ctx context.Context, fdst fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error { + return runSyncCopyMove(ctx, fdst, fdst, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs, true) +} + +// MoveDir moves fsrc into fdst +func MoveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error { + fi := filter.GetConfig(ctx) + if operations.Same(fdst, fsrc) { + fs.Errorf(fdst, "Nothing to do as source and destination are the same") + return nil + } + + // First attempt to use DirMover if exists, same Fs and no filters are active + if fdstDirMove := fdst.Features().DirMove; fdstDirMove != nil && operations.SameConfig(fsrc, fdst) && fi.InActive() { + if operations.SkipDestructive(ctx, fdst, "server-side directory move") { + return nil + } + fs.Debugf(fdst, "Using server-side directory move") + err := fdstDirMove(ctx, fsrc, "", "") + switch err { + case fs.ErrorCantDirMove, fs.ErrorDirExists: + fs.Infof(fdst, "Server side directory move failed - fallback to file moves: %v", err) + case nil: + fs.Infof(fdst, "Server side directory move succeeded") + return nil + default: + err = fs.CountError(ctx, err) + fs.Errorf(fdst, "Server side directory move failed: %v", err) + return err + } + } + + // Otherwise move the files one by one + return moveDir(ctx, fdst, fsrc, deleteEmptySrcDirs, copyEmptySrcDirs) +} diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go new file mode 100644 index 0000000..9865ecc --- /dev/null +++ b/fs/sync/sync_test.go @@ -0,0 +1,3108 @@ +// Test sync/copy/move + +package sync + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "runtime" + "sort" + "strings" + "testing" + "time" + + mutex "sync" // renamed as "sync" already in use + + _ "github.com/mewsen/rclone-studip-backend-oot/backend/studip" + _ "github.com/rclone/rclone/backend/all" // import all backends + "github.com/rclone/rclone/cmd/bisync/bilib" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/lib/transform" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/text/unicode/norm" +) + +// Some times used in the tests +var ( + t1 = fstest.Time("2001-02-03T04:05:06.499999999Z") + t2 = fstest.Time("2011-12-25T12:59:59.123456789Z") + t3 = fstest.Time("2011-12-30T12:59:59.000000000Z") +) + +// TestMain drives the tests +func TestMain(m *testing.M) { + fstest.TestMain(m) +} + +// Check dry run is working +func TestCopyWithDryRun(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + r.Mkdir(ctx, r.Fremote) + + ci.DryRun = true + ctx = predictDstFromLogger(ctx) + err := CopyDir(ctx, r.Fremote, r.Flocal, false) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) // error expected here because dry-run + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t) +} + +// Now without dry run +func TestCopy(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + _, err := operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t2) + if err != nil && !errors.Is(err, fs.ErrorNotImplemented) { + require.NoError(t, err) + } + r.Mkdir(ctx, r.Fremote) + + ctx = predictDstFromLogger(ctx) + err = CopyDir(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) + + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, "sub dir") +} + +func testCopyMetadata(t *testing.T, createEmptySrcDirs bool) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + r := fstest.NewRun(t) + features := r.Fremote.Features() + + if !features.ReadMetadata && !features.WriteMetadata && !features.UserMetadata && + !features.ReadDirMetadata && !features.WriteDirMetadata && !features.UserDirMetadata { + t.Skip("Skipping as metadata not supported") + } + + if createEmptySrcDirs && !features.CanHaveEmptyDirectories { + t.Skip("Skipping as can't have empty directories") + } + + const content = "hello metadata world!" + const dirPath = "metadata sub dir" + const emptyDirPath = "empty metadata sub dir" + const filePath = dirPath + "/hello metadata world" + + fileMetadata := fs.Metadata{ + // System metadata supported by all backends + "mtime": t1.Format(time.RFC3339Nano), + // User metadata + "potato": "jersey", + } + + dirMetadata := fs.Metadata{ + // System metadata supported by all backends + "mtime": t2.Format(time.RFC3339Nano), + // User metadata + "potato": "king edward", + } + + // Make the directory with metadata - may fall back to Mkdir + _, err := operations.MkdirMetadata(ctx, r.Flocal, dirPath, dirMetadata) + require.NoError(t, err) + + // Make the empty directory with metadata - may fall back to Mkdir + _, err = operations.MkdirMetadata(ctx, r.Flocal, emptyDirPath, dirMetadata) + require.NoError(t, err) + + // Upload the file with metadata + in := io.NopCloser(bytes.NewBufferString(content)) + _, err = operations.Rcat(ctx, r.Flocal, filePath, in, t1, fileMetadata) + require.NoError(t, err) + file1 := fstest.NewItem(filePath, content, t1) + + // Reset the time of the directory + _, err = operations.SetDirModTime(ctx, r.Flocal, nil, dirPath, t2) + if err != nil && !errors.Is(err, fs.ErrorNotImplemented) { + require.NoError(t, err) + } + + ctx = predictDstFromLogger(ctx) + err = CopyDir(ctx, r.Fremote, r.Flocal, createEmptySrcDirs) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) + + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, dirPath) + + // Check that the metadata on the directory and file is correct + if features.WriteMetadata && features.ReadMetadata { + fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewObject(ctx, t, r.Fremote, filePath), fileMetadata) + } + if features.WriteDirMetadata && features.ReadDirMetadata { + fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, dirPath), dirMetadata) + } + if !createEmptySrcDirs { + // dir must not exist + _, err := fstest.NewDirectoryRetries(ctx, t, r.Fremote, emptyDirPath, 1) + assert.Error(t, err, "Not expecting to find empty directory") + assert.True(t, errors.Is(err, fs.ErrorDirNotFound), fmt.Sprintf("expecting wrapped %#v not: %#v", fs.ErrorDirNotFound, err)) + } else { + // dir must exist + dir := fstest.NewDirectory(ctx, t, r.Fremote, emptyDirPath) + if features.ReadDirMetadata { + fstest.CheckEntryMetadata(ctx, t, r.Fremote, dir, dirMetadata) + } + } +} + +func TestCopyMetadata(t *testing.T) { + testCopyMetadata(t, true) +} + +func TestCopyMetadataNoEmptyDirs(t *testing.T) { + testCopyMetadata(t, false) +} + +func TestCopyMissingDirectory(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + r.Mkdir(ctx, r.Fremote) + + nonExistingFs, err := fs.NewFs(ctx, "/non-existing") + if err != nil { + t.Fatal(err) + } + + ctx = predictDstFromLogger(ctx) + err = CopyDir(ctx, r.Fremote, nonExistingFs, false) + require.Error(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) +} + +// Now with --no-traverse +func TestCopyNoTraverse(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + ci.NoTraverse = true + + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + + ctx = predictDstFromLogger(ctx) + err := CopyDir(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) +} + +func TestCopyNoTraverseDeadlock(t *testing.T) { + r := fstest.NewRun(t) + if !r.Fremote.Features().IsLocal { + t.Skip("Only runs on local") + } + const nFiles = 200 + t1 := fstest.Time("2001-02-03T04:05:06.499999999Z") + + // Create lots of source files. + items := make([]fstest.Item, nFiles) + for i := range items { + name := fmt.Sprintf("file%d.txt", i) + items[i] = r.WriteFile(name, fmt.Sprintf("content%d", i), t1) + } + r.CheckLocalItems(t, items...) + + // Set --no-traverse + ctx, ci := fs.AddConfig(context.Background()) + ci.NoTraverse = true + + // Initial copy to establish destination. + require.NoError(t, CopyDir(ctx, r.Fremote, r.Flocal, false)) + r.CheckRemoteItems(t, items...) + + // Second copy which shouldn't deadlock + require.NoError(t, CopyDir(ctx, r.Flocal, r.Fremote, false)) + r.CheckRemoteItems(t, items...) +} + +// Now with --check-first +func TestCopyCheckFirst(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + ci.CheckFirst = true + + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + + ctx = predictDstFromLogger(ctx) + err := CopyDir(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) +} + +// Now with --no-traverse +func TestSyncNoTraverse(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + ci.NoTraverse = true + + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) +} + +// Test copy with depth +func TestCopyWithDepth(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + file2 := r.WriteFile("hello world2", "hello world2", t2) + + // Check the MaxDepth too + ci.MaxDepth = 1 + + ctx = predictDstFromLogger(ctx) + err := CopyDir(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file2) +} + +// Test copy with files from +func testCopyWithFilesFrom(t *testing.T, noTraverse bool) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + file1 := r.WriteFile("potato2", "hello world", t1) + file2 := r.WriteFile("hello world2", "hello world2", t2) + + // Set the --files-from equivalent + f, err := filter.NewFilter(nil) + require.NoError(t, err) + require.NoError(t, f.AddFile("potato2")) + require.NoError(t, f.AddFile("notfound")) + + // Change the active filter + ctx = filter.ReplaceConfig(ctx, f) + + ci.NoTraverse = noTraverse + + ctx = predictDstFromLogger(ctx) + err = CopyDir(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file1) +} +func TestCopyWithFilesFrom(t *testing.T) { testCopyWithFilesFrom(t, false) } +func TestCopyWithFilesFromAndNoTraverse(t *testing.T) { testCopyWithFilesFrom(t, true) } + +// Test copy empty directories +func TestCopyEmptyDirectories(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + _, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2/sub sub dir2", t2) + require.NoError(t, err) + _, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir2", t2) + require.NoError(t, err) + r.Mkdir(ctx, r.Fremote) + + // Set the modtime on "sub dir" to something specific + // Without this it fails on the CI and in VirtualBox with variances of up to 10mS + _, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1) + require.NoError(t, err) + + ctx = predictDstFromLogger(ctx) + err = CopyDir(ctx, r.Fremote, r.Flocal, true) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckRemoteListing( + t, + []fstest.Item{ + file1, + }, + []string{ + "sub dir", + "sub dir2", + "sub dir2/sub sub dir2", + }, + ) + + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, "sub dir", "sub dir2", "sub dir2/sub sub dir2") +} + +// Test copy empty directories when we are configured not to create them +func TestCopyNoEmptyDirectories(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + err := operations.Mkdir(ctx, r.Flocal, "sub dir2") + require.NoError(t, err) + _, err = operations.MkdirModTime(ctx, r.Flocal, "sub dir2/sub sub dir2", t2) + require.NoError(t, err) + r.Mkdir(ctx, r.Fremote) + + err = CopyDir(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + + r.CheckRemoteListing( + t, + []fstest.Item{ + file1, + }, + []string{ + "sub dir", + }, + ) +} + +// Test move empty directories +func TestMoveEmptyDirectories(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + _, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2) + require.NoError(t, err) + subDir := fstest.NewDirectory(ctx, t, r.Flocal, "sub dir") + subDirT := subDir.ModTime(ctx) + r.Mkdir(ctx, r.Fremote) + + ctx = predictDstFromLogger(ctx) + err = MoveDir(ctx, r.Fremote, r.Flocal, false, true) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckRemoteListing( + t, + []fstest.Item{ + file1, + }, + []string{ + "sub dir", + "sub dir2", + }, + ) + + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, "sub dir2") + // Note that "sub dir" mod time is updated when file1 is deleted from it + // So check it more manually + got := fstest.NewDirectory(ctx, t, r.Fremote, "sub dir") + fstest.CheckDirModTime(ctx, t, r.Fremote, got, subDirT) +} + +// Test that --no-update-dir-modtime is working +func TestSyncNoUpdateDirModtime(t *testing.T) { + r := fstest.NewRun(t) + if r.Fremote.Features().DirSetModTime == nil { + t.Skip("Skipping test as backend does not support DirSetModTime") + } + + ctx, ci := fs.AddConfig(context.Background()) + ci.NoUpdateDirModTime = true + const name = "sub dir no update dir modtime" + + // Set the modtime on name to something specific + _, err := operations.MkdirModTime(ctx, r.Flocal, name, t1) + require.NoError(t, err) + + // Create the remote directory with the current time + require.NoError(t, r.Fremote.Mkdir(ctx, name)) + + // Read its modification time + wantT := fstest.NewDirectory(ctx, t, r.Fremote, name).ModTime(ctx) + + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckRemoteListing( + t, + []fstest.Item{}, + []string{ + name, + }, + ) + + // Read the new directory modification time - it should not have changed + gotT := fstest.NewDirectory(ctx, t, r.Fremote, name).ModTime(ctx) + fstest.AssertTimeEqualWithPrecision(t, name, wantT, gotT, r.Fremote.Precision()) +} + +// Test move empty directories when we are not configured to create them +func TestMoveNoEmptyDirectories(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + err := operations.Mkdir(ctx, r.Flocal, "sub dir2") + require.NoError(t, err) + r.Mkdir(ctx, r.Fremote) + + err = MoveDir(ctx, r.Fremote, r.Flocal, false, false) + require.NoError(t, err) + + r.CheckRemoteListing( + t, + []fstest.Item{ + file1, + }, + []string{ + "sub dir", + }, + ) +} + +// Test sync empty directories +func TestSyncEmptyDirectories(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + _, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2) + require.NoError(t, err) + + // Set the modtime on "sub dir" to something specific + // Without this it fails on the CI and in VirtualBox with variances of up to 10mS + _, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1) + require.NoError(t, err) + + r.Mkdir(ctx, r.Fremote) + + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckRemoteListing( + t, + []fstest.Item{ + file1, + }, + []string{ + "sub dir", + "sub dir2", + }, + ) + + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, "sub dir", "sub dir2") +} + +// Test delayed mod time setting +func TestSyncSetDelayedModTimes(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + + if !r.Fremote.Features().DirModTimeUpdatesOnWrite { + t.Skip("Backend doesn't have DirModTimeUpdatesOnWrite set") + } + + // Create directories without timestamps + require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d1/e1/f1")) + require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b2/c1/d1/e1/f1")) + require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d2/e1/f1")) + require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d2/e1/f2")) + + dirs := []string{ + "a1", + "a1/b1", + "a1/b1/c1", + "a1/b1/c1/d1", + "a1/b1/c1/d1/e1", + "a1/b1/c1/d1/e1/f1", + "a1/b1/c1/d2", + "a1/b1/c1/d2/e1", + "a1/b1/c1/d2/e1/f1", + "a1/b1/c1/d2/e1/f2", + "a1/b2", + "a1/b2/c1", + "a1/b2/c1/d1", + "a1/b2/c1/d1/e1", + "a1/b2/c1/d1/e1/f1", + } + r.CheckLocalListing(t, []fstest.Item{}, dirs) + + // Timestamp the directories in reverse order + ts := t1 + for i := len(dirs) - 1; i >= 0; i-- { + dir := dirs[i] + _, err := operations.SetDirModTime(ctx, r.Flocal, nil, dir, ts) + require.NoError(t, err) + ts = ts.Add(time.Minute) + } + + r.Mkdir(ctx, r.Fremote) + + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, true) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckRemoteListing(t, []fstest.Item{}, dirs) + + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, dirs...) +} + +// Test sync empty directories when we are not configured to create them +func TestSyncNoEmptyDirectories(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + err := operations.Mkdir(ctx, r.Flocal, "sub dir2") + require.NoError(t, err) + r.Mkdir(ctx, r.Fremote) + + err = Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + + r.CheckRemoteListing( + t, + []fstest.Item{ + file1, + }, + []string{ + "sub dir", + }, + ) +} + +// Test a server-side copy if possible, or the backup path if not +func TestServerSideCopy(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1) + r.CheckRemoteItems(t, file1) + + FremoteCopy, _, finaliseCopy, err := fstest.RandomRemote() + require.NoError(t, err) + defer finaliseCopy() + t.Logf("Server side copy (if possible) %v -> %v", r.Fremote, FremoteCopy) + + ctx = predictDstFromLogger(ctx) + err = CopyDir(ctx, FremoteCopy, r.Fremote, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + fstest.CheckItems(t, FremoteCopy, file1) +} + +// Test copying a file over itself +func TestCopyOverSelf(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1) + r.CheckRemoteItems(t, file1) + file2 := r.WriteFile("sub dir/hello world", "hello world again", t2) + r.CheckLocalItems(t, file2) + + ctx = predictDstFromLogger(ctx) + err := CopyDir(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckRemoteItems(t, file2) +} + +// Test server-side copying a file over itself +func TestServerSideCopyOverSelf(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1) + r.CheckRemoteItems(t, file1) + + FremoteCopy, _, finaliseCopy, err := fstest.RandomRemote() + require.NoError(t, err) + defer finaliseCopy() + t.Logf("Server side copy (if possible) %v -> %v", r.Fremote, FremoteCopy) + + ctx = predictDstFromLogger(ctx) + err = CopyDir(ctx, FremoteCopy, r.Fremote, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, FremoteCopy, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + fstest.CheckItems(t, FremoteCopy, file1) + + file2 := r.WriteObject(ctx, "sub dir/hello world", "hello world again", t2) + r.CheckRemoteItems(t, file2) + + ctx = predictDstFromLogger(ctx) + err = CopyDir(ctx, FremoteCopy, r.Fremote, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, FremoteCopy, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + fstest.CheckItems(t, FremoteCopy, file2) +} + +// Test moving a file over itself +func TestMoveOverSelf(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1) + r.CheckRemoteItems(t, file1) + file2 := r.WriteFile("sub dir/hello world", "hello world again", t2) + r.CheckLocalItems(t, file2) + + ctx = predictDstFromLogger(ctx) + err := MoveDir(ctx, r.Fremote, r.Flocal, false, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckLocalItems(t) + r.CheckRemoteItems(t, file2) +} + +// Test server-side moving a file over itself +func TestServerSideMoveOverSelf(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1) + r.CheckRemoteItems(t, file1) + + FremoteCopy, _, finaliseCopy, err := fstest.RandomRemote() + require.NoError(t, err) + defer finaliseCopy() + t.Logf("Server side copy (if possible) %v -> %v", r.Fremote, FremoteCopy) + + ctx = predictDstFromLogger(ctx) + err = CopyDir(ctx, FremoteCopy, r.Fremote, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, FremoteCopy, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + fstest.CheckItems(t, FremoteCopy, file1) + + file2 := r.WriteObject(ctx, "sub dir/hello world", "hello world again", t2) + r.CheckRemoteItems(t, file2) + + // ctx = predictDstFromLogger(ctx) + err = MoveDir(ctx, FremoteCopy, r.Fremote, false, false) + require.NoError(t, err) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) // not currently supported + r.CheckRemoteItems(t) + fstest.CheckItems(t, FremoteCopy, file2) + + // check that individual file moves also work without MoveDir + file3 := r.WriteObject(ctx, "sub dir/hello world", "hello world a third time", t3) + r.CheckRemoteItems(t, file3) + + ctx = predictDstFromLogger(ctx) + fs.Debugf(nil, "testing file moves") + err = moveDir(ctx, FremoteCopy, r.Fremote, false, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, FremoteCopy, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckRemoteItems(t) + fstest.CheckItems(t, FremoteCopy, file3) +} + +// Check that if the local file doesn't exist when we copy it up, +// nothing happens to the remote file +func TestCopyAfterDelete(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1) + r.CheckLocalItems(t) + r.CheckRemoteItems(t, file1) + + err := operations.Mkdir(ctx, r.Flocal, "") + require.NoError(t, err) + + ctx = predictDstFromLogger(ctx) + err = CopyDir(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalItems(t) + r.CheckRemoteItems(t, file1) +} + +// Check the copy downloading a file +func TestCopyRedownload(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1) + r.CheckRemoteItems(t, file1) + + ctx = predictDstFromLogger(ctx) + err := CopyDir(ctx, r.Flocal, r.Fremote, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + // Test with combined precision of local and remote as we copied it there and back + r.CheckLocalListing(t, []fstest.Item{file1}, nil) +} + +// Create a file and sync it. Change the last modified date and resync. +// If we're only doing sync by size and checksum, we expect nothing to +// to be transferred on the second sync. +func TestSyncBasedOnCheckSum(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + ci.CheckSum = true + + file1 := r.WriteFile("check sum", "-", t1) + r.CheckLocalItems(t, file1) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + // We should have transferred exactly one file. + assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers()) + r.CheckRemoteItems(t, file1) + + // Change last modified date only + file2 := r.WriteFile("check sum", "-", t2) + r.CheckLocalItems(t, file2) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + // We should have transferred no files + assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers()) + r.CheckLocalItems(t, file2) + r.CheckRemoteItems(t, file1) +} + +// Create a file and sync it. Change the last modified date and the +// file contents but not the size. If we're only doing sync by size +// only, we expect nothing to be transferred on the second sync. +func TestSyncSizeOnly(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + ci.SizeOnly = true + + file1 := r.WriteFile("sizeonly", "potato", t1) + r.CheckLocalItems(t, file1) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + // We should have transferred exactly one file. + assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers()) + r.CheckRemoteItems(t, file1) + + // Update mtime, md5sum but not length of file + file2 := r.WriteFile("sizeonly", "POTATO", t2) + r.CheckLocalItems(t, file2) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + // We should have transferred no files + assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers()) + r.CheckLocalItems(t, file2) + r.CheckRemoteItems(t, file1) +} + +// Create a file and sync it. Keep the last modified date but change +// the size. With --ignore-size we expect nothing to be +// transferred on the second sync. +func TestSyncIgnoreSize(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + ci.IgnoreSize = true + + file1 := r.WriteFile("ignore-size", "contents", t1) + r.CheckLocalItems(t, file1) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + // We should have transferred exactly one file. + assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers()) + r.CheckRemoteItems(t, file1) + + // Update size but not date of file + file2 := r.WriteFile("ignore-size", "longer contents but same date", t1) + r.CheckLocalItems(t, file2) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + // We should have transferred no files + assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers()) + r.CheckLocalItems(t, file2) + r.CheckRemoteItems(t, file1) +} + +func TestSyncIgnoreTimes(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + file1 := r.WriteBoth(ctx, "existing", "potato", t1) + r.CheckRemoteItems(t, file1) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + // We should have transferred exactly 0 files because the + // files were identical. + assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers()) + + ci.IgnoreTimes = true + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + // We should have transferred exactly one file even though the + // files were identical. + assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers()) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) +} + +func TestSyncIgnoreExisting(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + file1 := r.WriteFile("existing", "potato", t1) + + ci.IgnoreExisting = true + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + // Change everything + r.WriteFile("existing", "newpotatoes", t2) + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + // Items should not change + r.CheckRemoteItems(t, file1) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) +} + +func TestSyncIgnoreErrors(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + ci.IgnoreErrors = true + file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1) + file2 := r.WriteObject(ctx, "b/potato", "SMALLER BUT SAME DATE", t2) + file3 := r.WriteBoth(ctx, "c/non empty space", "AhHa!", t2) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "d")) + + r.CheckLocalListing( + t, + []fstest.Item{ + file1, + file3, + }, + []string{ + "a", + "c", + }, + ) + r.CheckRemoteListing( + t, + []fstest.Item{ + file2, + file3, + }, + []string{ + "b", + "c", + "d", + }, + ) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + _ = fs.CountError(ctx, errors.New("boom")) + assert.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false)) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalListing( + t, + []fstest.Item{ + file1, + file3, + }, + []string{ + "a", + "c", + }, + ) + r.CheckRemoteListing( + t, + []fstest.Item{ + file1, + file3, + }, + []string{ + "a", + "c", + }, + ) +} + +func TestSyncAfterChangingModtimeOnly(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + file1 := r.WriteFile("empty space", "-", t2) + file2 := r.WriteObject(ctx, "empty space", "-", t1) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) + + ci.DryRun = true + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) + + ci.DryRun = false + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) +} + +func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + if r.Fremote.Hashes().Count() == 0 { + t.Logf("Can't check this if no hashes supported") + return + } + + ci.NoUpdateModTime = true + + file1 := r.WriteFile("empty space", "-", t2) + file2 := r.WriteObject(ctx, "empty space", "-", t1) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) +} + +func TestSyncDoesntUpdateModtime(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + if fs.GetModifyWindow(ctx, r.Fremote) == fs.ModTimeNotSupported { + t.Skip("Can't run this test on fs which doesn't support mod time") + } + + file1 := r.WriteFile("foo", "foo", t2) + file2 := r.WriteObject(ctx, "foo", "bar", t1) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) + + // We should have transferred exactly one file, not set the mod time + assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers()) +} + +func TestSyncAfterAddingAFile(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteBoth(ctx, "empty space", "-", t2) + file2 := r.WriteFile("potato", "------------------------------------------------------------", t3) + + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file1) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file1, file2) +} + +func TestSyncAfterChangingFilesSizeOnly(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteObject(ctx, "potato", "------------------------------------------------------------", t3) + file2 := r.WriteFile("potato", "smaller but same date", t3) + r.CheckRemoteItems(t, file1) + r.CheckLocalItems(t, file2) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckLocalItems(t, file2) + r.CheckRemoteItems(t, file2) +} + +// Sync after changing a file's contents, changing modtime but length +// remaining the same +func TestSyncAfterChangingContentsOnly(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + var file1 fstest.Item + if r.Fremote.Precision() == fs.ModTimeNotSupported { + t.Logf("ModTimeNotSupported so forcing file to be a different size") + file1 = r.WriteObject(ctx, "potato", "different size to make sure it syncs", t3) + } else { + file1 = r.WriteObject(ctx, "potato", "smaller but same date", t3) + } + file2 := r.WriteFile("potato", "SMALLER BUT SAME DATE", t2) + r.CheckRemoteItems(t, file1) + r.CheckLocalItems(t, file2) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckLocalItems(t, file2) + r.CheckRemoteItems(t, file2) +} + +// Sync after removing a file and adding a file --dry-run +func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + file1 := r.WriteFile("potato2", "------------------------------------------------------------", t1) + file2 := r.WriteObject(ctx, "potato", "SMALLER BUT SAME DATE", t2) + file3 := r.WriteBoth(ctx, "empty space", "-", t2) + + ci.DryRun = true + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + ci.DryRun = false + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalItems(t, file3, file1) + r.CheckRemoteItems(t, file3, file2) +} + +// Sync after removing a file and adding a file +func testSyncAfterRemovingAFileAndAddingAFile(ctx context.Context, t *testing.T) { + r := fstest.NewRun(t) + file1 := r.WriteFile("potato2", "------------------------------------------------------------", t1) + file2 := r.WriteObject(ctx, "potato", "SMALLER BUT SAME DATE", t2) + file3 := r.WriteBoth(ctx, "empty space", "-", t2) + r.CheckRemoteItems(t, file2, file3) + r.CheckLocalItems(t, file1, file3) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckLocalItems(t, file1, file3) + r.CheckRemoteItems(t, file1, file3) +} + +func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) { + testSyncAfterRemovingAFileAndAddingAFile(context.Background(), t) +} + +// Sync after removing a file and adding a file +func testSyncAfterRemovingAFileAndAddingAFileSubDir(ctx context.Context, t *testing.T) { + r := fstest.NewRun(t) + file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1) + file2 := r.WriteObject(ctx, "b/potato", "SMALLER BUT SAME DATE", t2) + file3 := r.WriteBoth(ctx, "c/non empty space", "AhHa!", t2) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "d")) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "d/e")) + + r.CheckLocalListing( + t, + []fstest.Item{ + file1, + file3, + }, + []string{ + "a", + "c", + }, + ) + r.CheckRemoteListing( + t, + []fstest.Item{ + file2, + file3, + }, + []string{ + "b", + "c", + "d", + "d/e", + }, + ) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalListing( + t, + []fstest.Item{ + file1, + file3, + }, + []string{ + "a", + "c", + }, + ) + r.CheckRemoteListing( + t, + []fstest.Item{ + file1, + file3, + }, + []string{ + "a", + "c", + }, + ) +} + +func TestSyncAfterRemovingAFileAndAddingAFileSubDir(t *testing.T) { + testSyncAfterRemovingAFileAndAddingAFileSubDir(context.Background(), t) +} + +// Sync after removing a file and adding a file with IO Errors +func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1) + file2 := r.WriteObject(ctx, "b/potato", "SMALLER BUT SAME DATE", t2) + file3 := r.WriteBoth(ctx, "c/non empty space", "AhHa!", t2) + require.NoError(t, operations.Mkdir(ctx, r.Fremote, "d")) + + r.CheckLocalListing( + t, + []fstest.Item{ + file1, + file3, + }, + []string{ + "a", + "c", + }, + ) + r.CheckRemoteListing( + t, + []fstest.Item{ + file2, + file3, + }, + []string{ + "b", + "c", + "d", + }, + ) + + ctx = predictDstFromLogger(ctx) + accounting.GlobalStats().ResetCounters() + _ = fs.CountError(ctx, errors.New("boom")) + err := Sync(ctx, r.Fremote, r.Flocal, false) + assert.Equal(t, fs.ErrorNotDeleting, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + accounting.GlobalStats().ResetCounters() + + r.CheckLocalListing( + t, + []fstest.Item{ + file1, + file3, + }, + []string{ + "a", + "c", + }, + ) + r.CheckRemoteListing( + t, + []fstest.Item{ + file1, + file2, + file3, + }, + []string{ + "a", + "b", + "c", + "d", + }, + ) +} + +// Sync test delete after +func TestSyncDeleteAfter(t *testing.T) { + ctx := context.Background() + ci := fs.GetConfig(ctx) + // This is the default so we've checked this already + // check it is the default + require.Equal(t, ci.DeleteMode, fs.DeleteModeAfter, "Didn't default to --delete-after") +} + +// Sync test delete during +func TestSyncDeleteDuring(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + ci.DeleteMode = fs.DeleteModeDuring + + testSyncAfterRemovingAFileAndAddingAFile(ctx, t) +} + +// Sync test delete before +func TestSyncDeleteBefore(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + ci.DeleteMode = fs.DeleteModeBefore + + testSyncAfterRemovingAFileAndAddingAFile(ctx, t) +} + +// Copy test delete before - shouldn't delete anything +func TestCopyDeleteBefore(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + ci.DeleteMode = fs.DeleteModeBefore + + file1 := r.WriteObject(ctx, "potato", "hopefully not deleted", t1) + file2 := r.WriteFile("potato2", "hopefully copied in", t1) + r.CheckRemoteItems(t, file1) + r.CheckLocalItems(t, file2) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := CopyDir(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckRemoteItems(t, file1, file2) + r.CheckLocalItems(t, file2) +} + +// Test with exclude +func TestSyncWithExclude(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) + file2 := r.WriteBoth(ctx, "empty space", "-", t2) + file3 := r.WriteFile("enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes + r.CheckRemoteItems(t, file1, file2) + r.CheckLocalItems(t, file1, file2, file3) + + fi, err := filter.NewFilter(nil) + require.NoError(t, err) + fi.Opt.MaxSize = 40 + ctx = filter.ReplaceConfig(ctx, fi) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckRemoteItems(t, file2, file1) + + // Now sync the other way round and check enormous doesn't get + // deleted as it is excluded from the sync + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Flocal, r.Fremote, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckLocalItems(t, file2, file1, file3) +} + +// Test with exclude and delete excluded +func TestSyncWithExcludeAndDeleteExcluded(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) // 60 bytes + file2 := r.WriteBoth(ctx, "empty space", "-", t2) + file3 := r.WriteBoth(ctx, "enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes + r.CheckRemoteItems(t, file1, file2, file3) + r.CheckLocalItems(t, file1, file2, file3) + + fi, err := filter.NewFilter(nil) + require.NoError(t, err) + fi.Opt.MaxSize = 40 + fi.Opt.DeleteExcluded = true + ctx = filter.ReplaceConfig(ctx, fi) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckRemoteItems(t, file2) + + // Check sync the other way round to make sure enormous gets + // deleted even though it is excluded + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Flocal, r.Fremote, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckLocalItems(t, file2) +} + +// Test with UpdateOlder set +func TestSyncWithUpdateOlder(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + if fs.GetModifyWindow(ctx, r.Fremote) == fs.ModTimeNotSupported { + t.Skip("Can't run this test on fs which doesn't support mod time") + } + t2plus := t2.Add(time.Second / 2) + t2minus := t2.Add(time.Second / 2) + oneF := r.WriteFile("one", "one", t1) + twoF := r.WriteFile("two", "two", t3) + threeF := r.WriteFile("three", "three", t2) + fourF := r.WriteFile("four", "four", t2) + fiveF := r.WriteFile("five", "five", t2) + r.CheckLocalItems(t, oneF, twoF, threeF, fourF, fiveF) + oneO := r.WriteObject(ctx, "one", "ONE", t2) + twoO := r.WriteObject(ctx, "two", "TWO", t2) + threeO := r.WriteObject(ctx, "three", "THREE", t2plus) + fourO := r.WriteObject(ctx, "four", "FOURFOUR", t2minus) + r.CheckRemoteItems(t, oneO, twoO, threeO, fourO) + + ci.UpdateOlder = true + ci.ModifyWindow = fs.Duration(fs.ModTimeNotSupported) + + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + r.CheckRemoteItems(t, oneO, twoF, threeO, fourF, fiveF) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) // no modtime + + if r.Fremote.Hashes().Count() == 0 { + t.Logf("Skip test with --checksum as no hashes supported") + return + } + + // now enable checksum + ci.CheckSum = true + + err = Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + r.CheckRemoteItems(t, oneO, twoF, threeF, fourF, fiveF) +} + +// Test with a max transfer duration +func testSyncWithMaxDuration(t *testing.T, cutoffMode fs.CutoffMode) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + if *fstest.RemoteName != "" { + t.Skip("Skipping test on non local remote") + } + r := fstest.NewRun(t) + + maxDuration := fs.Duration(250 * time.Millisecond) + ci.MaxDuration = maxDuration + ci.CutoffMode = cutoffMode + ci.CheckFirst = true + ci.OrderBy = "size" + ci.Transfers = 1 + ci.Checkers = 1 + bytesPerSecond := 10 * 1024 + accounting.TokenBucket.SetBwLimit(fs.BwPair{Tx: fs.SizeSuffix(bytesPerSecond), Rx: fs.SizeSuffix(bytesPerSecond)}) + defer accounting.TokenBucket.SetBwLimit(fs.BwPair{Tx: -1, Rx: -1}) + + // write one small file which we expect to transfer and one big one which we don't + file1 := r.WriteFile("file1", string(make([]byte, 16)), t1) + file2 := r.WriteFile("file2", string(make([]byte, 50*1024)), t1) + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t) + + if runtime.GOOS == "darwin" { + r.Flocal.Features().Disable("Copy") // macOS cloning is too fast for this test! + if r.Fremote.Features().IsLocal { + r.Fremote.Features().Disable("Copy") // macOS cloning is too fast for this test! + } + } + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) // not currently supported (but tests do pass for CutoffModeSoft) + startTime := time.Now() + err := Sync(ctx, r.Fremote, r.Flocal, false) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.True(t, errors.Is(err, ErrorMaxDurationReached)) + + if cutoffMode == fs.CutoffModeHard { + r.CheckRemoteItems(t, file1) + assert.Equal(t, int64(1), accounting.GlobalStats().GetTransfers()) + } else { + r.CheckRemoteItems(t, file1, file2) + assert.Equal(t, int64(2), accounting.GlobalStats().GetTransfers()) + } + + elapsed := time.Since(startTime) + const maxTransferTime = 20 * time.Second + + what := fmt.Sprintf("expecting elapsed time %v between %v and %v", elapsed, maxDuration, maxTransferTime) + assert.True(t, elapsed >= time.Duration(maxDuration), what) + assert.True(t, elapsed < maxTransferTime, what) +} + +func TestSyncWithMaxDuration(t *testing.T) { + t.Run("Hard", func(t *testing.T) { + testSyncWithMaxDuration(t, fs.CutoffModeHard) + }) + t.Run("Soft", func(t *testing.T) { + testSyncWithMaxDuration(t, fs.CutoffModeSoft) + }) +} + +// Test with TrackRenames set +func TestSyncWithTrackRenames(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + ci.TrackRenames = true + defer func() { + ci.TrackRenames = false + }() + + haveHash := r.Fremote.Hashes().Overlap(r.Flocal.Hashes()).GetOne() != hash.None + canTrackRenames := haveHash && operations.CanServerSideMove(r.Fremote) + t.Logf("Can track renames: %v", canTrackRenames) + + f1 := r.WriteFile("potato", "Potato Content", t1) + f2 := r.WriteFile("yam", "Yam Content", t2) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false)) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckRemoteItems(t, f1, f2) + r.CheckLocalItems(t, f1, f2) + + // Now rename locally. + f2 = r.RenameFile(f2, "yaml") + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false)) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckRemoteItems(t, f1, f2) + + // Check we renamed something if we should have + if canTrackRenames { + renames := accounting.GlobalStats().Renames(0) + assert.Equal(t, canTrackRenames, renames != 0, fmt.Sprintf("canTrackRenames=%v, renames=%d", canTrackRenames, renames)) + } +} + +func TestParseRenamesStrategyModtime(t *testing.T) { + for _, test := range []struct { + in string + want trackRenamesStrategy + wantErr bool + }{ + {"", 0, false}, + {"modtime", trackRenamesStrategyModtime, false}, + {"hash", trackRenamesStrategyHash, false}, + {"size", 0, false}, + {"modtime,hash", trackRenamesStrategyModtime | trackRenamesStrategyHash, false}, + {"hash,modtime,size", trackRenamesStrategyModtime | trackRenamesStrategyHash, false}, + {"size,boom", 0, true}, + } { + got, err := parseTrackRenamesStrategy(test.in) + assert.Equal(t, test.want, got, test.in) + assert.Equal(t, test.wantErr, err != nil, test.in) + } +} + +func TestRenamesStrategyModtime(t *testing.T) { + both := trackRenamesStrategyHash | trackRenamesStrategyModtime + hash := trackRenamesStrategyHash + modTime := trackRenamesStrategyModtime + + assert.True(t, both.hash()) + assert.True(t, both.modTime()) + assert.True(t, hash.hash()) + assert.False(t, hash.modTime()) + assert.False(t, modTime.hash()) + assert.True(t, modTime.modTime()) +} + +func TestSyncWithTrackRenamesStrategyModtime(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + ci.TrackRenames = true + ci.TrackRenamesStrategy = "modtime" + + canTrackRenames := operations.CanServerSideMove(r.Fremote) && r.Fremote.Precision() != fs.ModTimeNotSupported + t.Logf("Can track renames: %v", canTrackRenames) + + f1 := r.WriteFile("potato", "Potato Content", t1) + f2 := r.WriteFile("yam", "Yam Content", t2) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false)) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckRemoteItems(t, f1, f2) + r.CheckLocalItems(t, f1, f2) + + // Now rename locally. + f2 = r.RenameFile(f2, "yaml") + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false)) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckRemoteItems(t, f1, f2) + + // Check we renamed something if we should have + if canTrackRenames { + renames := accounting.GlobalStats().Renames(0) + assert.Equal(t, canTrackRenames, renames != 0, fmt.Sprintf("canTrackRenames=%v, renames=%d", canTrackRenames, renames)) + } +} + +func TestSyncWithTrackRenamesStrategyLeaf(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + ci.TrackRenames = true + ci.TrackRenamesStrategy = "leaf" + + canTrackRenames := operations.CanServerSideMove(r.Fremote) && r.Fremote.Precision() != fs.ModTimeNotSupported + t.Logf("Can track renames: %v", canTrackRenames) + + f1 := r.WriteFile("potato", "Potato Content", t1) + f2 := r.WriteFile("sub/yam", "Yam Content", t2) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false)) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckRemoteItems(t, f1, f2) + r.CheckLocalItems(t, f1, f2) + + // Now rename locally. + f2 = r.RenameFile(f2, "yam") + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + require.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false)) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckRemoteItems(t, f1, f2) + + // Check we renamed something if we should have + if canTrackRenames { + renames := accounting.GlobalStats().Renames(0) + assert.Equal(t, canTrackRenames, renames != 0, fmt.Sprintf("canTrackRenames=%v, renames=%d", canTrackRenames, renames)) + } +} + +func toyFileTransfers(r *fstest.Run) int64 { + remote := r.Fremote.Name() + transfers := 1 + if strings.HasPrefix(remote, "TestChunker") && strings.HasSuffix(remote, "S3") { + transfers++ // Extra Copy because S3 emulates Move as Copy+Delete. + } + return int64(transfers) +} + +// Test a server-side move if possible, or the backup path if not +func testServerSideMove(ctx context.Context, t *testing.T, r *fstest.Run, withFilter, testDeleteEmptyDirs bool) { + FremoteMove, _, finaliseMove, err := fstest.RandomRemote() + require.NoError(t, err) + defer finaliseMove() + + file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) + file2 := r.WriteBoth(ctx, "empty space", "-", t2) + file3u := r.WriteBoth(ctx, "potato3", "------------------------------------------------------------ UPDATED", t2) + + if testDeleteEmptyDirs { + err := operations.Mkdir(ctx, r.Fremote, "tomatoDir") + require.NoError(t, err) + } + + r.CheckRemoteItems(t, file2, file1, file3u) + + t.Logf("Server side move (if possible) %v -> %v", r.Fremote, FremoteMove) + + // Write just one file in the new remote + r.WriteObjectTo(ctx, FremoteMove, "empty space", "-", t2, false) + file3 := r.WriteObjectTo(ctx, FremoteMove, "potato3", "------------------------------------------------------------", t1, false) + fstest.CheckItems(t, FremoteMove, file2, file3) + + // Do server-side move + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) // not currently supported -- doesn't list all contents of dir. + err = MoveDir(ctx, FremoteMove, r.Fremote, testDeleteEmptyDirs, false) + require.NoError(t, err) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + if withFilter { + r.CheckRemoteItems(t, file2) + } else { + r.CheckRemoteItems(t) + } + + if testDeleteEmptyDirs { + r.CheckRemoteListing(t, nil, []string{}) + } + + fstest.CheckItems(t, FremoteMove, file2, file1, file3u) + + // Create a new empty remote for stuff to be moved into + FremoteMove2, _, finaliseMove2, err := fstest.RandomRemote() + require.NoError(t, err) + defer finaliseMove2() + + if testDeleteEmptyDirs { + err := operations.Mkdir(ctx, FremoteMove, "tomatoDir") + require.NoError(t, err) + } + + // Move it back to a new empty remote, dst does not exist this time + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err = MoveDir(ctx, FremoteMove2, FremoteMove, testDeleteEmptyDirs, false) + require.NoError(t, err) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + if withFilter { + fstest.CheckItems(t, FremoteMove2, file1, file3u) + fstest.CheckItems(t, FremoteMove, file2) + } else { + fstest.CheckItems(t, FremoteMove2, file2, file1, file3u) + fstest.CheckItems(t, FremoteMove) + } + + if testDeleteEmptyDirs { + fstest.CheckListingWithPrecision(t, FremoteMove, nil, []string{}, fs.GetModifyWindow(ctx, r.Fremote)) + } +} + +// Test MoveDir on Local +func TestServerSideMoveLocal(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + f1 := r.WriteFile("dir1/file1.txt", "hello", t1) + f2 := r.WriteFile("dir2/file2.txt", "hello again", t2) + r.CheckLocalItems(t, f1, f2) + + dir1, err := fs.NewFs(ctx, r.Flocal.Root()+"/dir1") + require.NoError(t, err) + dir2, err := fs.NewFs(ctx, r.Flocal.Root()+"/dir2") + require.NoError(t, err) + err = MoveDir(ctx, dir2, dir1, false, false) + require.NoError(t, err) +} + +// Test move +func TestMoveWithDeleteEmptySrcDirs(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + file2 := r.WriteFile("nested/sub dir/file", "nested", t1) + r.Mkdir(ctx, r.Fremote) + + // run move with --delete-empty-src-dirs + ctx = predictDstFromLogger(ctx) + err := MoveDir(ctx, r.Fremote, r.Flocal, true, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalListing( + t, + nil, + []string{}, + ) + r.CheckRemoteItems(t, file1, file2) +} + +func TestMoveWithoutDeleteEmptySrcDirs(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + file2 := r.WriteFile("nested/sub dir/file", "nested", t1) + r.Mkdir(ctx, r.Fremote) + + ctx = predictDstFromLogger(ctx) + err := MoveDir(ctx, r.Fremote, r.Flocal, false, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + r.CheckLocalListing( + t, + nil, + []string{ + "sub dir", + "nested", + "nested/sub dir", + }, + ) + r.CheckRemoteItems(t, file1, file2) +} + +func TestMoveWithIgnoreExisting(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + file1 := r.WriteFile("existing", "potato", t1) + file2 := r.WriteFile("existing-b", "tomato", t1) + + ci.IgnoreExisting = true + + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err := MoveDir(ctx, r.Fremote, r.Flocal, false, false) + require.NoError(t, err) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckLocalListing( + t, + []fstest.Item{}, + []string{}, + ) + r.CheckRemoteListing( + t, + []fstest.Item{ + file1, + file2, + }, + []string{}, + ) + + // Recreate first file with modified content + file1b := r.WriteFile("existing", "newpotatoes", t2) + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err = MoveDir(ctx, r.Fremote, r.Flocal, false, false) + require.NoError(t, err) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + // Source items should still exist in modified state + r.CheckLocalListing( + t, + []fstest.Item{ + file1b, + }, + []string{}, + ) + // Dest items should not have changed + r.CheckRemoteListing( + t, + []fstest.Item{ + file1, + file2, + }, + []string{}, + ) +} + +// Test a server-side move if possible, or the backup path if not +func TestServerSideMove(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + testServerSideMove(ctx, t, r, false, false) +} + +// Test a server-side move if possible, or the backup path if not +func TestServerSideMoveWithFilter(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + + fi, err := filter.NewFilter(nil) + require.NoError(t, err) + fi.Opt.MinSize = 40 + ctx = filter.ReplaceConfig(ctx, fi) + + testServerSideMove(ctx, t, r, true, false) +} + +// Test a server-side move if possible +func TestServerSideMoveDeleteEmptySourceDirs(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + testServerSideMove(ctx, t, r, false, true) +} + +// Test a server-side move with overlap +func TestServerSideMoveOverlap(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + + if r.Fremote.Features().DirMove != nil { + t.Skip("Skipping test as remote supports DirMove") + } + + subRemoteName := r.FremoteName + "/rclone-move-test" + FremoteMove, err := fs.NewFs(ctx, subRemoteName) + require.NoError(t, err) + + file1 := r.WriteObject(ctx, "potato2", "------------------------------------------------------------", t1) + r.CheckRemoteItems(t, file1) + + // Subdir move with no filters should return ErrorCantMoveOverlapping + // ctx = predictDstFromLogger(ctx) + err = MoveDir(ctx, FremoteMove, r.Fremote, false, false) + assert.EqualError(t, err, fs.ErrorOverlapping.Error()) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + // Now try with a filter which should also fail with ErrorCantMoveOverlapping + fi, err := filter.NewFilter(nil) + require.NoError(t, err) + fi.Opt.MinSize = 40 + ctx = filter.ReplaceConfig(ctx, fi) + + // ctx = predictDstFromLogger(ctx) + err = MoveDir(ctx, FremoteMove, r.Fremote, false, false) + assert.EqualError(t, err, fs.ErrorOverlapping.Error()) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) +} + +// Test a sync with overlap +func TestSyncOverlap(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + + subRemoteName := r.FremoteName + "/rclone-sync-test" + FremoteSync, err := fs.NewFs(ctx, subRemoteName) + require.NoError(t, err) + + checkErr := func(err error) { + require.Error(t, err) + assert.True(t, fserrors.IsFatalError(err)) + assert.Equal(t, fs.ErrorOverlapping.Error(), err.Error()) + } + + ctx = predictDstFromLogger(ctx) + checkErr(Sync(ctx, FremoteSync, r.Fremote, false)) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + ctx = predictDstFromLogger(ctx) + checkErr(Sync(ctx, r.Fremote, FremoteSync, false)) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + ctx = predictDstFromLogger(ctx) + checkErr(Sync(ctx, r.Fremote, r.Fremote, false)) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + ctx = predictDstFromLogger(ctx) + checkErr(Sync(ctx, FremoteSync, FremoteSync, false)) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) +} + +// Test a sync with filtered overlap +func TestSyncOverlapWithFilter(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + + fi, err := filter.NewFilter(nil) + require.NoError(t, err) + require.NoError(t, fi.Add(false, "/rclone-sync-test/")) + require.NoError(t, fi.Add(false, "*/layer2/")) + fi.Opt.ExcludeFile = []string{".ignore"} + filterCtx := filter.ReplaceConfig(ctx, fi) + + subRemoteName := r.FremoteName + "/rclone-sync-test" + FremoteSync, err := fs.NewFs(ctx, subRemoteName) + require.NoError(t, FremoteSync.Mkdir(ctx, "")) + require.NoError(t, err) + + subRemoteName2 := r.FremoteName + "/rclone-sync-test-include/layer2" + FremoteSync2, err := fs.NewFs(ctx, subRemoteName2) + require.NoError(t, FremoteSync2.Mkdir(ctx, "")) + require.NoError(t, err) + + subRemoteName3 := r.FremoteName + "/rclone-sync-test-ignore-file" + FremoteSync3, err := fs.NewFs(ctx, subRemoteName3) + require.NoError(t, FremoteSync3.Mkdir(ctx, "")) + require.NoError(t, err) + r.WriteObject(context.Background(), "rclone-sync-test-ignore-file/.ignore", "-", t1) + + checkErr := func(err error) { + require.Error(t, err) + assert.True(t, fserrors.IsFatalError(err)) + assert.Equal(t, fs.ErrorOverlapping.Error(), err.Error()) + accounting.GlobalStats().ResetCounters() + } + + checkNoErr := func(err error) { + require.NoError(t, err) + } + + accounting.GlobalStats().ResetCounters() + filterCtx = predictDstFromLogger(filterCtx) + checkNoErr(Sync(filterCtx, FremoteSync, r.Fremote, false)) + checkErr(Sync(ctx, FremoteSync, r.Fremote, false)) + checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + checkErr(Sync(ctx, r.Fremote, FremoteSync, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + checkErr(Sync(filterCtx, r.Fremote, r.Fremote, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + checkErr(Sync(ctx, r.Fremote, r.Fremote, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + checkErr(Sync(filterCtx, FremoteSync, FremoteSync, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + checkErr(Sync(ctx, FremoteSync, FremoteSync, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + + checkNoErr(Sync(filterCtx, FremoteSync2, r.Fremote, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + checkErr(Sync(ctx, FremoteSync2, r.Fremote, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync2, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + checkErr(Sync(ctx, r.Fremote, FremoteSync2, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + checkErr(Sync(filterCtx, FremoteSync2, FremoteSync2, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + checkErr(Sync(ctx, FremoteSync2, FremoteSync2, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + + checkNoErr(Sync(filterCtx, FremoteSync3, r.Fremote, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + checkErr(Sync(ctx, FremoteSync3, r.Fremote, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + // Destination is excluded so this test makes no sense + // checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync3, false)) + checkErr(Sync(ctx, r.Fremote, FremoteSync3, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + checkErr(Sync(filterCtx, FremoteSync3, FremoteSync3, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) + filterCtx = predictDstFromLogger(filterCtx) + checkErr(Sync(ctx, FremoteSync3, FremoteSync3, false)) + testLoggerVsLsf(filterCtx, r.Fremote, r.Flocal, operations.GetLoggerOpt(filterCtx).JSON, t) +} + +// Test with CompareDest set +func TestSyncCompareDest(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + ci.CompareDest = []string{r.FremoteName + "/CompareDest"} + + fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") + require.NoError(t, err) + + // check empty dest, empty compare + file1 := r.WriteFile("one", "one", t1) + r.CheckLocalItems(t, file1) + + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) // not currently supported due to duplicate equal() checks + err = Sync(ctx, fdst, r.Flocal, false) + // testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + file1dst := file1 + file1dst.Path = "dst/one" + + r.CheckRemoteItems(t, file1dst) + + // check old dest, empty compare + file1b := r.WriteFile("one", "onet2", t2) + r.CheckRemoteItems(t, file1dst) + r.CheckLocalItems(t, file1b) + + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err = Sync(ctx, fdst, r.Flocal, false) + // testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + file1bdst := file1b + file1bdst.Path = "dst/one" + + r.CheckRemoteItems(t, file1bdst) + + // check old dest, new compare + file3 := r.WriteObject(ctx, "dst/one", "one", t1) + file2 := r.WriteObject(ctx, "CompareDest/one", "onet2", t2) + file1c := r.WriteFile("one", "onet2", t2) + r.CheckRemoteItems(t, file2, file3) + r.CheckLocalItems(t, file1c) + + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err = Sync(ctx, fdst, r.Flocal, false) + // testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckRemoteItems(t, file2, file3) + + // check empty dest, new compare + file4 := r.WriteObject(ctx, "CompareDest/two", "two", t2) + file5 := r.WriteFile("two", "two", t2) + r.CheckRemoteItems(t, file2, file3, file4) + r.CheckLocalItems(t, file1c, file5) + + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err = Sync(ctx, fdst, r.Flocal, false) + // testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckRemoteItems(t, file2, file3, file4) + + // check new dest, new compare + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err = Sync(ctx, fdst, r.Flocal, false) + // testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckRemoteItems(t, file2, file3, file4) + + // Work out if we actually have hashes for uploaded files + haveHash := false + if ht := fdst.Hashes().GetOne(); ht != hash.None { + file2obj, err := fdst.NewObject(ctx, "one") + if err == nil { + file2objHash, err := file2obj.Hash(ctx, ht) + if err == nil { + haveHash = file2objHash != "" + } + } + } + + // check new dest, new compare, src timestamp differs + // + // we only check this if we the file we uploaded previously + // actually has a hash otherwise the differing timestamp is + // always copied. + if haveHash { + file5b := r.WriteFile("two", "two", t3) + r.CheckLocalItems(t, file1c, file5b) + + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err = Sync(ctx, fdst, r.Flocal, false) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckRemoteItems(t, file2, file3, file4) + } else { + t.Log("No hash on uploaded file so skipping compare timestamp test") + } + + // check empty dest, old compare + file5c := r.WriteFile("two", "twot3", t3) + r.CheckRemoteItems(t, file2, file3, file4) + r.CheckLocalItems(t, file1c, file5c) + + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err = Sync(ctx, fdst, r.Flocal, false) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + file5cdst := file5c + file5cdst.Path = "dst/two" + + r.CheckRemoteItems(t, file2, file3, file4, file5cdst) +} + +// Test with multiple CompareDest +func TestSyncMultipleCompareDest(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + precision := fs.GetModifyWindow(ctx, r.Fremote, r.Flocal) + + ci.CompareDest = []string{r.FremoteName + "/pre-dest1", r.FremoteName + "/pre-dest2"} + + // check empty dest, new compare + fsrc1 := r.WriteFile("1", "1", t1) + fsrc2 := r.WriteFile("2", "2", t1) + fsrc3 := r.WriteFile("3", "3", t1) + r.CheckLocalItems(t, fsrc1, fsrc2, fsrc3) + + fdest1 := r.WriteObject(ctx, "pre-dest1/1", "1", t1) + fdest2 := r.WriteObject(ctx, "pre-dest2/2", "2", t1) + r.CheckRemoteItems(t, fdest1, fdest2) + + accounting.GlobalStats().ResetCounters() + fdst, err := fs.NewFs(ctx, r.FremoteName+"/dest") + require.NoError(t, err) + // ctx = predictDstFromLogger(ctx) + require.NoError(t, Sync(ctx, fdst, r.Flocal, false)) + // testLoggerVsLsf(ctx, fdst, operations.GetLoggerOpt(ctx).JSON, t) + + fdest3 := fsrc3 + fdest3.Path = "dest/3" + + fstest.CheckItemsWithPrecision(t, fdst, precision, fsrc3) + r.CheckRemoteItems(t, fdest1, fdest2, fdest3) +} + +// Test with CopyDest set +func TestSyncCopyDest(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + if r.Fremote.Features().Copy == nil { + t.Skip("Skipping test as remote does not support server-side copy") + } + + ci.CopyDest = []string{r.FremoteName + "/CopyDest"} + + fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") + require.NoError(t, err) + + // check empty dest, empty copy + file1 := r.WriteFile("one", "one", t1) + r.CheckLocalItems(t, file1) + + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err = Sync(ctx, fdst, r.Flocal, false) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) // not currently supported + require.NoError(t, err) + + file1dst := file1 + file1dst.Path = "dst/one" + + r.CheckRemoteItems(t, file1dst) + + // check old dest, empty copy + file1b := r.WriteFile("one", "onet2", t2) + r.CheckRemoteItems(t, file1dst) + r.CheckLocalItems(t, file1b) + + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err = Sync(ctx, fdst, r.Flocal, false) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + file1bdst := file1b + file1bdst.Path = "dst/one" + + r.CheckRemoteItems(t, file1bdst) + + // check old dest, new copy, backup-dir + + ci.BackupDir = r.FremoteName + "/BackupDir" + + file3 := r.WriteObject(ctx, "dst/one", "one", t1) + file2 := r.WriteObject(ctx, "CopyDest/one", "onet2", t2) + file1c := r.WriteFile("one", "onet2", t2) + r.CheckRemoteItems(t, file2, file3) + r.CheckLocalItems(t, file1c) + + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err = Sync(ctx, fdst, r.Flocal, false) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + file2dst := file2 + file2dst.Path = "dst/one" + file3.Path = "BackupDir/one" + + r.CheckRemoteItems(t, file2, file2dst, file3) + ci.BackupDir = "" + + // check empty dest, new copy + file4 := r.WriteObject(ctx, "CopyDest/two", "two", t2) + file5 := r.WriteFile("two", "two", t2) + r.CheckRemoteItems(t, file2, file2dst, file3, file4) + r.CheckLocalItems(t, file1c, file5) + + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + err = Sync(ctx, fdst, r.Flocal, false) + require.NoError(t, err) + + file4dst := file4 + file4dst.Path = "dst/two" + + r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst) + + // check new dest, new copy + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err = Sync(ctx, fdst, r.Flocal, false) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst) + + // check empty dest, old copy + file6 := r.WriteObject(ctx, "CopyDest/three", "three", t2) + file7 := r.WriteFile("three", "threet3", t3) + r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6) + r.CheckLocalItems(t, file1c, file5, file7) + + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err = Sync(ctx, fdst, r.Flocal, false) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + file7dst := file7 + file7dst.Path = "dst/three" + + r.CheckRemoteItems(t, file2, file2dst, file3, file4, file4dst, file6, file7dst) +} + +// Test with BackupDir set +func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeepExtension bool) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + if !operations.CanServerSideMove(r.Fremote) { + t.Skip("Skipping test as remote does not support server-side move") + } + r.Mkdir(ctx, r.Fremote) + + if backupDir != "" { + ci.BackupDir = r.FremoteName + "/" + backupDir + backupDir += "/" + } else { + ci.BackupDir = "" + backupDir = "dst/" + // Exclude the suffix from the sync otherwise the sync + // deletes the old backup files + flt, err := filter.NewFilter(nil) + require.NoError(t, err) + require.NoError(t, flt.AddRule("- *"+suffix)) + // Change the active filter + ctx = filter.ReplaceConfig(ctx, flt) + } + ci.Suffix = suffix + ci.SuffixKeepExtension = suffixKeepExtension + + // Make the setup so we have one, two, three in the dest + // and one (different), two (same) in the source + file1 := r.WriteObject(ctx, "dst/one", "one", t1) + file2 := r.WriteObject(ctx, "dst/two", "two", t1) + file3 := r.WriteObject(ctx, "dst/three.txt", "three", t1) + file2a := r.WriteFile("two", "two", t1) + file1a := r.WriteFile("one", "oneA", t2) + + r.CheckRemoteItems(t, file1, file2, file3) + r.CheckLocalItems(t, file1a, file2a) + + fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") + require.NoError(t, err) + + accounting.GlobalStats().ResetCounters() + err = Sync(ctx, fdst, r.Flocal, false) + require.NoError(t, err) + + // one should be moved to the backup dir and the new one installed + file1.Path = backupDir + "one" + suffix + file1a.Path = "dst/one" + // two should be unchanged + // three should be moved to the backup dir + if suffixKeepExtension { + file3.Path = backupDir + "three" + suffix + ".txt" + } else { + file3.Path = backupDir + "three.txt" + suffix + } + + r.CheckRemoteItems(t, file1, file2, file3, file1a) + + // Now check what happens if we do it again + // Restore a different three and update one in the source + file3a := r.WriteObject(ctx, "dst/three.txt", "threeA", t2) + file1b := r.WriteFile("one", "oneBB", t3) + r.CheckRemoteItems(t, file1, file2, file3, file1a, file3a) + + // This should delete three and overwrite one again, checking + // the files got overwritten correctly in backup-dir + accounting.GlobalStats().ResetCounters() + err = Sync(ctx, fdst, r.Flocal, false) + require.NoError(t, err) + + // one should be moved to the backup dir and the new one installed + file1a.Path = backupDir + "one" + suffix + file1b.Path = "dst/one" + // two should be unchanged + // three should be moved to the backup dir + if suffixKeepExtension { + file3a.Path = backupDir + "three" + suffix + ".txt" + } else { + file3a.Path = backupDir + "three.txt" + suffix + } + + r.CheckRemoteItems(t, file1b, file2, file3a, file1a) +} + +func TestSyncBackupDir(t *testing.T) { + testSyncBackupDir(t, "backup", "", false) +} + +func TestSyncBackupDirWithSuffix(t *testing.T) { + testSyncBackupDir(t, "backup", ".bak", false) +} + +func TestSyncBackupDirWithSuffixKeepExtension(t *testing.T) { + testSyncBackupDir(t, "backup", "-2019-01-01", true) +} + +func TestSyncBackupDirSuffixOnly(t *testing.T) { + testSyncBackupDir(t, "", ".bak", false) +} + +// Test with Suffix set +func testSyncSuffix(t *testing.T, suffix string, suffixKeepExtension bool) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + if !operations.CanServerSideMove(r.Fremote) { + t.Skip("Skipping test as remote does not support server-side move") + } + r.Mkdir(ctx, r.Fremote) + + ci.Suffix = suffix + ci.SuffixKeepExtension = suffixKeepExtension + + // Make the setup so we have one, two, three in the dest + // and one (different), two (same) in the source + file1 := r.WriteObject(ctx, "dst/one", "one", t1) + file2 := r.WriteObject(ctx, "dst/two", "two", t1) + file3 := r.WriteObject(ctx, "dst/three.txt", "three", t1) + file2a := r.WriteFile("two", "two", t1) + file1a := r.WriteFile("one", "oneA", t2) + file3a := r.WriteFile("three.txt", "threeA", t1) + + r.CheckRemoteItems(t, file1, file2, file3) + r.CheckLocalItems(t, file1a, file2a, file3a) + + fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst") + require.NoError(t, err) + + accounting.GlobalStats().ResetCounters() + err = operations.CopyFile(ctx, fdst, r.Flocal, "one", "one") + require.NoError(t, err) + err = operations.CopyFile(ctx, fdst, r.Flocal, "two", "two") + require.NoError(t, err) + err = operations.CopyFile(ctx, fdst, r.Flocal, "three.txt", "three.txt") + require.NoError(t, err) + + // one should be moved to the backup dir and the new one installed + file1.Path = "dst/one" + suffix + file1a.Path = "dst/one" + // two should be unchanged + // three should be moved to the backup dir + if suffixKeepExtension { + file3.Path = "dst/three" + suffix + ".txt" + } else { + file3.Path = "dst/three.txt" + suffix + } + file3a.Path = "dst/three.txt" + + r.CheckRemoteItems(t, file1, file2, file3, file1a, file3a) + + // Now check what happens if we do it again + // Restore a different three and update one in the source + file3b := r.WriteFile("three.txt", "threeBDifferentSize", t3) + file1b := r.WriteFile("one", "oneBB", t3) + r.CheckRemoteItems(t, file1, file2, file3, file1a, file3a) + + // This should delete three and overwrite one again, checking + // the files got overwritten correctly in backup-dir + accounting.GlobalStats().ResetCounters() + err = operations.CopyFile(ctx, fdst, r.Flocal, "one", "one") + require.NoError(t, err) + err = operations.CopyFile(ctx, fdst, r.Flocal, "two", "two") + require.NoError(t, err) + err = operations.CopyFile(ctx, fdst, r.Flocal, "three.txt", "three.txt") + require.NoError(t, err) + + // one should be moved to the backup dir and the new one installed + file1a.Path = "dst/one" + suffix + file1b.Path = "dst/one" + // two should be unchanged + // three should be moved to the backup dir + if suffixKeepExtension { + file3a.Path = "dst/three" + suffix + ".txt" + } else { + file3a.Path = "dst/three.txt" + suffix + } + file3b.Path = "dst/three.txt" + + r.CheckRemoteItems(t, file1b, file3b, file2, file3a, file1a) +} +func TestSyncSuffix(t *testing.T) { testSyncSuffix(t, ".bak", false) } +func TestSyncSuffixKeepExtension(t *testing.T) { testSyncSuffix(t, "-2019-01-01", true) } + +// Check we can sync two files with differing UTF-8 representations +func TestSyncUTFNorm(t *testing.T) { + ctx := context.Background() + if runtime.GOOS == "darwin" { + t.Skip("Can't test UTF normalization on OS X") + } + + r := fstest.NewRun(t) + + // Two strings with different unicode normalization (from OS X) + Encoding1 := "Testêé" + Encoding2 := "Testêé" + assert.NotEqual(t, Encoding1, Encoding2) + assert.Equal(t, norm.NFC.String(Encoding1), norm.NFC.String(Encoding2)) + + file1 := r.WriteFile(Encoding1, "This is a test", t1) + r.CheckLocalItems(t, file1) + + file2 := r.WriteObject(ctx, Encoding2, "This is a old test", t2) + r.CheckRemoteItems(t, file2) + + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) // can't test this on macOS + require.NoError(t, err) + + // We should have transferred exactly one file, but kept the + // normalized state of the file. + assert.Equal(t, toyFileTransfers(r), accounting.GlobalStats().GetTransfers()) + r.CheckLocalItems(t, file1) + file1.Path = file2.Path + r.CheckRemoteItems(t, file1) +} + +// Test --immutable +func TestSyncImmutable(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + ci.Immutable = true + + // Create file on source + file1 := r.WriteFile("existing", "potato", t1) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t) + + // Should succeed + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file1) + + // Modify file data and timestamp on source + file2 := r.WriteFile("existing", "tomatoes", t2) + r.CheckLocalItems(t, file2) + r.CheckRemoteItems(t, file1) + + // Should fail with ErrorImmutableModified and not modify local or remote files + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, false) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + assert.EqualError(t, err, fs.ErrorImmutableModified.Error()) + r.CheckLocalItems(t, file2) + r.CheckRemoteItems(t, file1) +} + +// Test --ignore-case-sync +func TestSyncIgnoreCase(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + // Only test if filesystems are case sensitive + if r.Fremote.Features().CaseInsensitive || r.Flocal.Features().CaseInsensitive { + t.Skip("Skipping test as local or remote are case-insensitive") + } + + ci.IgnoreCaseSync = true + + // Create files with different filename casing + file1 := r.WriteFile("existing", "potato", t1) + r.CheckLocalItems(t, file1) + file2 := r.WriteObject(ctx, "EXISTING", "potato", t1) + r.CheckRemoteItems(t, file2) + + // Should not copy files that are differently-cased but otherwise identical + accounting.GlobalStats().ResetCounters() + // ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) // can't test this on macOS + require.NoError(t, err) + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, file2) +} + +// Test --fix-case +func TestFixCase(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + r := fstest.NewRun(t) + + // Only test if remote is case insensitive + if !r.Fremote.Features().CaseInsensitive { + t.Skip("Skipping test as local or remote are case-sensitive") + } + + ci.FixCase = true + + // Create files with different filename casing + file1a := r.WriteFile("existing", "potato", t1) + file1b := r.WriteFile("existingbutdifferent", "donut", t1) + file1c := r.WriteFile("subdira/subdirb/subdirc/hello", "donut", t1) + file1d := r.WriteFile("subdira/subdirb/subdirc/subdird/filewithoutcasedifferences", "donut", t1) + r.CheckLocalItems(t, file1a, file1b, file1c, file1d) + file2a := r.WriteObject(ctx, "EXISTING", "potato", t1) + file2b := r.WriteObject(ctx, "EXISTINGBUTDIFFERENT", "lemonade", t1) + file2c := r.WriteObject(ctx, "SUBDIRA/subdirb/SUBDIRC/HELLO", "lemonade", t1) + file2d := r.WriteObject(ctx, "SUBDIRA/subdirb/SUBDIRC/subdird/filewithoutcasedifferences", "lemonade", t1) + r.CheckRemoteItems(t, file2a, file2b, file2c, file2d) + + // Should force rename of dest file that is differently-cased + accounting.GlobalStats().ResetCounters() + err := Sync(ctx, r.Fremote, r.Flocal, false) + require.NoError(t, err) + r.CheckLocalItems(t, file1a, file1b, file1c, file1d) + r.CheckRemoteItems(t, file1a, file1b, file1c, file1d) +} + +// Test that aborting on --max-transfer works +func TestMaxTransfer(t *testing.T) { + ctx := context.Background() + ctx, ci := fs.AddConfig(ctx) + ci.MaxTransfer = 3 * 1024 + ci.Transfers = 1 + ci.Checkers = 1 + ci.CutoffMode = fs.CutoffModeHard + + test := func(t *testing.T, cutoff fs.CutoffMode) { + r := fstest.NewRun(t) + ci.CutoffMode = cutoff + + if r.Fremote.Name() != "local" { + t.Skip("This test only runs on local") + } + + // Create file on source + file1 := r.WriteFile("file1", string(make([]byte, 5*1024)), t1) + file2 := r.WriteFile("file2", string(make([]byte, 2*1024)), t1) + file3 := r.WriteFile("file3", string(make([]byte, 3*1024)), t1) + r.CheckLocalItems(t, file1, file2, file3) + r.CheckRemoteItems(t) + + if runtime.GOOS == "darwin" { + // disable server-side copies as they don't count towards transfer size stats + r.Flocal.Features().Disable("Copy") + if r.Fremote.Features().IsLocal { + r.Fremote.Features().Disable("Copy") + } + } + + accounting.GlobalStats().ResetCounters() + + // ctx = predictDstFromLogger(ctx) // not currently supported + err := Sync(ctx, r.Fremote, r.Flocal, false) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + expectedErr := fserrors.FsError(accounting.ErrorMaxTransferLimitReachedFatal) + if cutoff != fs.CutoffModeHard { + expectedErr = accounting.ErrorMaxTransferLimitReachedGraceful + } + fserrors.Count(expectedErr) + assert.Equal(t, expectedErr, err) + } + + t.Run("Hard", func(t *testing.T) { test(t, fs.CutoffModeHard) }) + t.Run("Soft", func(t *testing.T) { test(t, fs.CutoffModeSoft) }) + t.Run("Cautious", func(t *testing.T) { test(t, fs.CutoffModeCautious) }) +} + +func testSyncConcurrent(t *testing.T, subtest string) { + const ( + NFILES = 20 + NCHECKERS = 4 + NTRANSFERS = 4 + ) + + ctx, ci := fs.AddConfig(context.Background()) + ci.Checkers = NCHECKERS + ci.Transfers = NTRANSFERS + + r := fstest.NewRun(t) + stats := accounting.GlobalStats() + + itemsBefore := []fstest.Item{} + itemsAfter := []fstest.Item{} + for i := range NFILES { + nameBoth := fmt.Sprintf("both%d", i) + nameOnly := fmt.Sprintf("only%d", i) + switch subtest { + case "delete": + fileBoth := r.WriteBoth(ctx, nameBoth, "potato", t1) + fileOnly := r.WriteObject(ctx, nameOnly, "potato", t1) + itemsBefore = append(itemsBefore, fileBoth, fileOnly) + itemsAfter = append(itemsAfter, fileBoth) + case "truncate": + fileBoth := r.WriteBoth(ctx, nameBoth, "potato", t1) + fileFull := r.WriteObject(ctx, nameOnly, "potato", t1) + fileEmpty := r.WriteFile(nameOnly, "", t1) + itemsBefore = append(itemsBefore, fileBoth, fileFull) + itemsAfter = append(itemsAfter, fileBoth, fileEmpty) + } + } + + r.CheckRemoteItems(t, itemsBefore...) + stats.ResetErrors() + ctx = predictDstFromLogger(ctx) + err := Sync(ctx, r.Fremote, r.Flocal, false) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + if errors.Is(err, fs.ErrorCantUploadEmptyFiles) { + t.Skipf("Skip test because remote cannot upload empty files") + } + assert.NoError(t, err, "Sync must not return a error") + assert.False(t, stats.Errored(), "Low level errors must not have happened") + r.CheckRemoteItems(t, itemsAfter...) +} + +func TestSyncConcurrentDelete(t *testing.T) { + testSyncConcurrent(t, "delete") +} + +func TestSyncConcurrentTruncate(t *testing.T) { + testSyncConcurrent(t, "truncate") +} + +// Test that sync replaces dir modtimes in dst if they've changed +func testSyncReplaceDirModTime(t *testing.T, copyEmptySrcDirs bool) { + accounting.GlobalStats().ResetCounters() + ctx, _ := fs.AddConfig(context.Background()) + r := fstest.NewRun(t) + + file1 := r.WriteFile("file1", "file1", t2) + file2 := r.WriteFile("test_dir1/file2", "file2", t2) + file3 := r.WriteFile("test_dir2/sub_dir/file3", "file3", t2) + r.CheckLocalItems(t, file1, file2, file3) + + _, err := operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t2) + require.NoError(t, err) + + // A directory that's empty on both src and dst + _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_on_remote", t2) + require.NoError(t, err) + _, err = operations.MkdirModTime(ctx, r.Fremote, "empty_on_remote", t2) + require.NoError(t, err) + + // set logging + // (this checks log output as DirModtime operations do not yet have stats, and r.CheckDirectoryModTimes also does not tell us what actions were taken) + oldLogLevel := fs.GetConfig(context.Background()).LogLevel + defer func() { fs.GetConfig(context.Background()).LogLevel = oldLogLevel }() // reset to old val after test + // need to do this as fs.Infof only respects the globalConfig + fs.GetConfig(context.Background()).LogLevel = fs.LogLevelInfo + + // First run + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + output := bilib.CaptureOutput(func() { + err := CopyDir(ctx, r.Fremote, r.Flocal, copyEmptySrcDirs) + require.NoError(t, err) + }) + require.NotNil(t, output) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + + // Save all dirs + dirs := []string{"test_dir1", "test_dir2", "test_dir2/sub_dir", "empty_on_remote"} + if copyEmptySrcDirs { + dirs = append(dirs, "empty_dir") + } + + // Change dir modtimes + for _, dir := range dirs { + _, err := operations.SetDirModTime(ctx, r.Flocal, nil, dir, t1) + if err != nil && !errors.Is(err, fs.ErrorNotImplemented) { + require.NoError(t, err) + } + } + + // Run again + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + output = bilib.CaptureOutput(func() { + err := CopyDir(ctx, r.Fremote, r.Flocal, copyEmptySrcDirs) + require.NoError(t, err) + }) + require.NotNil(t, output) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckLocalItems(t, file1, file2, file3) + r.CheckRemoteItems(t, file1, file2, file3) + + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, dirs...) +} + +func TestSyncReplaceDirModTime(t *testing.T) { + testSyncReplaceDirModTime(t, false) +} + +func TestSyncReplaceDirModTimeWithEmptyDirs(t *testing.T) { + testSyncReplaceDirModTime(t, true) +} + +// Tests that nothing is transferred when src and dst already match +// Run the same sync twice, ensure no action is taken the second time +func testNothingToTransfer(t *testing.T, copyEmptySrcDirs bool) { + accounting.GlobalStats().ResetCounters() + ctx, _ := fs.AddConfig(context.Background()) + r := fstest.NewRun(t) + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + file2 := r.WriteFile("sub dir2/very/very/very/very/very/nested/subdir/hello world", "hello world", t1) + r.CheckLocalItems(t, file1, file2) + _, err := operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t2) + if err != nil && !errors.Is(err, fs.ErrorNotImplemented) { + require.NoError(t, err) + } + r.Mkdir(ctx, r.Fremote) + _, err = operations.MkdirModTime(ctx, r.Fremote, "sub dir", t3) + require.NoError(t, err) + + // set logging + // (this checks log output as DirModtime operations do not yet have stats, and r.CheckDirectoryModTimes also does not tell us what actions were taken) + oldLogLevel := fs.GetConfig(context.Background()).LogLevel + defer func() { fs.GetConfig(context.Background()).LogLevel = oldLogLevel }() // reset to old val after test + // need to do this as fs.Infof only respects the globalConfig + fs.GetConfig(context.Background()).LogLevel = fs.LogLevelInfo + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + output := bilib.CaptureOutput(func() { + err = CopyDir(ctx, r.Fremote, r.Flocal, copyEmptySrcDirs) + require.NoError(t, err) + }) + require.NotNil(t, output) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file1, file2) + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, "sub dir", "sub dir2", "sub dir2/very", "sub dir2/very/very", "sub dir2/very/very/very/very/very/nested/subdir") + + // check that actions were taken + assert.True(t, strings.Contains(string(output), "Copied"), `expected to find at least one "Copied" log: `+string(output)) + if r.Fremote.Features().DirSetModTime != nil || r.Fremote.Features().MkdirMetadata != nil { + assert.True(t, strings.Contains(string(output), "Set directory modification time"), `expected to find at least one "Set directory modification time" log: `+string(output)) + } + assert.False(t, strings.Contains(string(output), "There was nothing to transfer"), `expected to find no "There was nothing to transfer" logs, but found one: `+string(output)) + assert.True(t, accounting.GlobalStats().GetTransfers() >= 2) + + // run it again and make sure no actions were taken + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + output = bilib.CaptureOutput(func() { + err = CopyDir(ctx, r.Fremote, r.Flocal, copyEmptySrcDirs) + require.NoError(t, err) + }) + require.NotNil(t, output) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t, file1, file2) + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, "sub dir", "sub dir2", "sub dir2/very", "sub dir2/very/very", "sub dir2/very/very/very/very/very/nested/subdir") + + // check that actions were NOT taken + assert.False(t, strings.Contains(string(output), "Copied"), `expected to find no "Copied" logs, but found one: `+string(output)) + if r.Fremote.Features().DirSetModTime != nil || r.Fremote.Features().MkdirMetadata != nil { + assert.False(t, strings.Contains(string(output), "Set directory modification time"), `expected to find no "Set directory modification time" logs, but found one: `+string(output)) + assert.False(t, strings.Contains(string(output), "Updated directory metadata"), `expected to find no "Updated directory metadata" logs, but found one: `+string(output)) + assert.False(t, strings.Contains(string(output), "directory"), `expected to find no "directory"-related logs, but found one: `+string(output)) // catch-all + } + assert.True(t, strings.Contains(string(output), "There was nothing to transfer"), `expected to find a "There was nothing to transfer" log: `+string(output)) + assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers()) + + // check nested empty dir behavior (FIXME: probably belongs in a separate test) + if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil { + return + } + file3 := r.WriteFile("sub dir2/sub dir3/hello world", "hello again, world", t1) + _, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir2", t1) + assert.NoError(t, err) + _, err = operations.SetDirModTime(ctx, r.Fremote, nil, "sub dir2", t1) + assert.NoError(t, err) + _, err = operations.MkdirModTime(ctx, r.Flocal, "sub dirEmpty/sub dirEmpty2", t2) + assert.NoError(t, err) + _, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dirEmpty", t2) + assert.NoError(t, err) + + accounting.GlobalStats().ResetCounters() + ctx = predictDstFromLogger(ctx) + output = bilib.CaptureOutput(func() { + err = CopyDir(ctx, r.Fremote, r.Flocal, copyEmptySrcDirs) + require.NoError(t, err) + }) + require.NotNil(t, output) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + r.CheckLocalItems(t, file1, file2, file3) + r.CheckRemoteItems(t, file1, file2, file3) + // Check that the modtimes of the directories are as expected + r.CheckDirectoryModTimes(t, "sub dir", "sub dir2", "sub dir2/very", "sub dir2/very/very", "sub dir2/very/very/very/very/very/nested/subdir", "sub dir2/sub dir3") + if copyEmptySrcDirs { + r.CheckDirectoryModTimes(t, "sub dirEmpty", "sub dirEmpty/sub dirEmpty2") + assert.True(t, strings.Contains(string(output), "sub dirEmpty:"), `expected to find at least one "sub dirEmpty:" log: `+string(output)) + } else { + assert.False(t, strings.Contains(string(output), "sub dirEmpty:"), `expected to find no "sub dirEmpty:" logs, but found one (empty dir was synced and shouldn't have been): `+string(output)) + } + assert.True(t, strings.Contains(string(output), "sub dir3:"), `expected to find at least one "sub dir3:" log: `+string(output)) + assert.False(t, strings.Contains(string(output), "sub dir2/very:"), `expected to find no "sub dir2/very:" logs, but found one (unmodified dir was marked modified): `+string(output)) +} + +func TestNothingToTransferWithEmptyDirs(t *testing.T) { + testNothingToTransfer(t, true) +} + +func TestNothingToTransferWithoutEmptyDirs(t *testing.T) { + testNothingToTransfer(t, false) +} + +// for testing logger: +func predictDstFromLogger(ctx context.Context) context.Context { + opt := operations.NewLoggerOpt() + var lock mutex.Mutex + + opt.LoggerFn = func(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) { + lock.Lock() + defer lock.Unlock() + + // ignore dirs for our purposes here + if err == fs.ErrorIsDir { + return + } + winner := operations.WinningSide(ctx, sigil, src, dst, err) + if winner.Obj != nil { + file := winner.Obj + obj, ok := file.(fs.ObjectInfo) + checksum := "" + timeFormat := "2006-01-02 15:04:05" + if ok { + if obj.Fs().Hashes().GetOne() == hash.MD5 { + // skip if no MD5 + checksum, _ = obj.Hash(ctx, hash.MD5) + } + timeFormat = operations.FormatForLSFPrecision(obj.Fs().Precision()) + } + errMsg := "" + if winner.Err != nil { + errMsg = ";" + winner.Err.Error() + } + operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), transform.Path(ctx, file.Remote(), false), errMsg) // TODO: should the transform be handled in the sync instead of here? + } + } + return operations.WithSyncLogger(ctx, opt) +} + +func DstLsf(ctx context.Context, Fremote fs.Fs) *bytes.Buffer { + opt := operations.ListJSONOpt{ + NoModTime: false, + NoMimeType: true, + DirsOnly: false, + FilesOnly: true, + Recurse: true, + ShowHash: true, + HashTypes: []string{"MD5"}, + } + + var list operations.ListFormat + + list.SetSeparator(";") + timeFormat := operations.FormatForLSFPrecision(Fremote.Precision()) + if Fremote.Precision() == fs.ModTimeNotSupported { + timeFormat = "none" + } + list.AddModTime(timeFormat) + list.AddHash(hash.MD5) + list.AddSize() + list.AddPath() + + out := new(bytes.Buffer) + + err := operations.ListJSON(ctx, Fremote, "", &opt, func(item *operations.ListJSONItem) error { + _, _ = fmt.Fprintln(out, list.Format(item)) + return nil + }) + if err != nil { + fs.Errorf(Fremote, "ListJSON error: %v", err) + } + + return out +} + +func LoggerMatchesLsf(logger, lsf *bytes.Buffer) error { + loggerSplit := bytes.Split(logger.Bytes(), []byte("\n")) + sort.SliceStable(loggerSplit, func(i int, j int) bool { return string(loggerSplit[i]) < string(loggerSplit[j]) }) + lsfSplit := bytes.Split(lsf.Bytes(), []byte("\n")) + sort.SliceStable(lsfSplit, func(i int, j int) bool { return string(lsfSplit[i]) < string(lsfSplit[j]) }) + + loggerJoined := bytes.Join(loggerSplit, []byte("\n")) + lsfJoined := bytes.Join(lsfSplit, []byte("\n")) + + if bytes.Equal(loggerJoined, lsfJoined) { + return nil + } + Diff(string(loggerJoined), string(lsfJoined)) + return fmt.Errorf("logger does not match lsf! \nlogger: \n%s \nlsf: \n%s", loggerJoined, lsfJoined) +} + +func Diff(rev1, rev2 string) { + fmt.Printf("Diff of %q and %q\n", "logger", "lsf") + cmd := exec.Command("bash", "-c", fmt.Sprintf(`diff <(echo "%s") <(echo "%s")`, rev1, rev2)) + out, _ := cmd.Output() + _, _ = os.Stdout.Write(out) +} + +func testLoggerVsLsf(ctx context.Context, fdst, fsrc fs.Fs, logger *bytes.Buffer, t *testing.T) { + var newlogger bytes.Buffer + canTestModtime := fs.GetModifyWindow(ctx, fdst) != fs.ModTimeNotSupported + canTestHash := fdst.Hashes().Contains(hash.MD5) + if !canTestHash || !canTestModtime { + loggerSplit := bytes.Split(logger.Bytes(), []byte("\n")) + for i, line := range loggerSplit { + elements := bytes.Split(line, []byte(";")) + if len(elements) >= 2 { + if !canTestModtime { + elements[0] = []byte("none") + } + if !canTestHash { + elements[1] = []byte("") + } + } + loggerSplit[i] = bytes.Join(elements, []byte(";")) + } + newlogger.Write(bytes.Join(loggerSplit, []byte("\n"))) + } else { + newlogger.Write(logger.Bytes()) + } + + if fsrc.Precision() == fdst.Precision() && fsrc.Hashes().Contains(hash.MD5) && canTestHash { + lsf := DstLsf(ctx, fdst) + err := LoggerMatchesLsf(&newlogger, lsf) + require.NoError(t, err) + } +} diff --git a/fs/sync/sync_transform_test.go b/fs/sync/sync_transform_test.go new file mode 100644 index 0000000..930386a --- /dev/null +++ b/fs/sync/sync_transform_test.go @@ -0,0 +1,515 @@ +// Test transform + +package sync + +import ( + "cmp" + "context" + "fmt" + "path" + "slices" + "strings" + "testing" + + _ "github.com/rclone/rclone/backend/all" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/lib/transform" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/text/unicode/norm" +) + +var debug = `` + +func TestTransform(t *testing.T) { + type args struct { + TransformOpt []string + TransformBackOpt []string + Lossless bool // whether the TransformBackAlgo is always losslessly invertible + } + tests := []struct { + name string + args args + }{ + {name: "NFC", args: args{ + TransformOpt: []string{"nfc"}, + TransformBackOpt: []string{"nfd"}, + Lossless: false, + }}, + {name: "NFD", args: args{ + TransformOpt: []string{"nfd"}, + TransformBackOpt: []string{"nfc"}, + Lossless: false, + }}, + {name: "base64", args: args{ + TransformOpt: []string{"base64encode"}, + TransformBackOpt: []string{"base64encode"}, + Lossless: false, + }}, + {name: "prefix", args: args{ + TransformOpt: []string{"prefix=PREFIX"}, + TransformBackOpt: []string{"trimprefix=PREFIX"}, + Lossless: true, + }}, + {name: "suffix", args: args{ + TransformOpt: []string{"suffix=SUFFIX"}, + TransformBackOpt: []string{"trimsuffix=SUFFIX"}, + Lossless: true, + }}, + {name: "truncate", args: args{ + TransformOpt: []string{"truncate=10"}, + TransformBackOpt: []string{"truncate=10"}, + Lossless: false, + }}, + {name: "encoder", args: args{ + TransformOpt: []string{"encoder=Colon,SquareBracket"}, + TransformBackOpt: []string{"decoder=Colon,SquareBracket"}, + Lossless: true, + }}, + {name: "ISO-8859-1", args: args{ + TransformOpt: []string{"ISO-8859-1"}, + TransformBackOpt: []string{"ISO-8859-1"}, + Lossless: false, + }}, + {name: "charmap", args: args{ + TransformOpt: []string{"all,charmap=ISO-8859-7"}, + TransformBackOpt: []string{"all,charmap=ISO-8859-7"}, + Lossless: false, + }}, + {name: "lowercase", args: args{ + TransformOpt: []string{"all,lowercase"}, + TransformBackOpt: []string{"all,lowercase"}, + Lossless: false, + }}, + {name: "ascii", args: args{ + TransformOpt: []string{"all,ascii"}, + TransformBackOpt: []string{"all,ascii"}, + Lossless: false, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + ctx := context.Background() + r.Mkdir(ctx, r.Flocal) + r.Mkdir(ctx, r.Fremote) + items := makeTestFiles(t, r, "dir1") + deleteDSStore(t, r) + r.CheckRemoteListing(t, items, nil) + r.CheckLocalListing(t, items, nil) + + err := transform.SetOptions(ctx, tt.args.TransformOpt...) + require.NoError(t, err) + + err = Sync(ctx, r.Fremote, r.Flocal, true) + assert.NoError(t, err) + compareNames(ctx, t, r, items) + + err = transform.SetOptions(ctx, tt.args.TransformBackOpt...) + require.NoError(t, err) + err = Sync(ctx, r.Fremote, r.Flocal, true) + assert.NoError(t, err) + compareNames(ctx, t, r, items) + + if tt.args.Lossless { + deleteDSStore(t, r) + r.CheckRemoteItems(t, items...) + } + }) + } +} + +const alphabet = "abcdefg123456789" + +var extras = []string{"apple", "banana", "appleappleapplebanana", "splitbananasplit"} + +func makeTestFiles(t *testing.T, r *fstest.Run, dir string) []fstest.Item { + t.Helper() + n := 0 + // Create test files + items := []fstest.Item{} + for _, c := range alphabet { + var out strings.Builder + for i := range rune(7) { + out.WriteRune(c + i) + } + fileName := path.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String())) + fileName = strings.ToValidUTF8(fileName, "") + fileName = strings.NewReplacer(":", "", "<", "", ">", "", "?", "").Replace(fileName) // remove characters illegal on windows + + if debug != "" { + fileName = debug + } + + item := r.WriteObject(context.Background(), fileName, fileName, t1) + r.WriteFile(fileName, fileName, t1) + items = append(items, item) + n++ + + if debug != "" { + break + } + } + + for _, extra := range extras { + item := r.WriteObject(context.Background(), extra, extra, t1) + r.WriteFile(extra, extra, t1) + items = append(items, item) + } + + return items +} + +func deleteDSStore(t *testing.T, r *fstest.Run) { + ctxDSStore, fi := filter.AddConfig(context.Background()) + err := fi.AddRule(`+ *.DS_Store`) + assert.NoError(t, err) + err = fi.AddRule(`- **`) + assert.NoError(t, err) + err = operations.Delete(ctxDSStore, r.Fremote) + assert.NoError(t, err) +} + +func compareNames(ctx context.Context, t *testing.T, r *fstest.Run, items []fstest.Item) { + var entries fs.DirEntries + + deleteDSStore(t, r) + err := walk.ListR(context.Background(), r.Fremote, "", true, -1, walk.ListObjects, func(e fs.DirEntries) error { + entries = append(entries, e...) + return nil + }) + assert.NoError(t, err) + entries = slices.DeleteFunc(entries, func(E fs.DirEntry) bool { // remove those pesky .DS_Store files + if strings.Contains(E.Remote(), ".DS_Store") { + err := operations.DeleteFile(context.Background(), E.(fs.Object)) + assert.NoError(t, err) + return true + } + return false + }) + require.Equal(t, len(items), entries.Len()) + + // sort by CONVERTED name + slices.SortStableFunc(items, func(a, b fstest.Item) int { + aConv := transform.Path(ctx, a.Path, false) + bConv := transform.Path(ctx, b.Path, false) + return cmp.Compare(aConv, bConv) + }) + slices.SortStableFunc(entries, func(a, b fs.DirEntry) int { + return cmp.Compare(a.Remote(), b.Remote()) + }) + + for i, e := range entries { + expect := transform.Path(ctx, items[i].Path, false) + msg := fmt.Sprintf("expected %v, got %v", detectEncoding(expect), detectEncoding(e.Remote())) + assert.Equal(t, expect, e.Remote(), msg) + } +} + +func detectEncoding(s string) string { + if norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) { + return "BOTH" + } + if !norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) { + return "NFD" + } + if norm.NFC.IsNormalString(s) && !norm.NFD.IsNormalString(s) { + return "NFC" + } + return "OTHER" +} + +func TestTransformCopy(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "all,suffix_keep_extension=_somesuffix") + require.NoError(t, err) + file1 := r.WriteFile("sub dir/hello world.txt", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("sub dir_somesuffix/hello world_somesuffix.txt", "hello world", t1)) +} + +func TestDoubleTransform(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic") + require.NoError(t, err) + file1 := r.WriteFile("toe/toe", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe", "hello world", t1)) +} + +func TestFileTag(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "file,prefix=tac", "file,prefix=tic") + require.NoError(t, err) + file1 := r.WriteFile("toe/toe/toe", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("toe/toe/tictactoe", "hello world", t1)) +} + +func TestNoTag(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "prefix=tac", "prefix=tic") + require.NoError(t, err) + file1 := r.WriteFile("toe/toe/toe", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("toe/toe/tictactoe", "hello world", t1)) +} + +func TestDirTag(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "dir,prefix=tac", "dir,prefix=tic") + require.NoError(t, err) + r.WriteFile("toe/toe/toe.txt", "hello world", t1) + _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1) + require.NoError(t, err) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"empty_dir", "toe", "toe/toe"}) + r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"}) +} + +func TestAllTag(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic") + require.NoError(t, err) + r.WriteFile("toe/toe/toe.txt", "hello world", t1) + _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1) + require.NoError(t, err) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"empty_dir", "toe", "toe/toe"}) + r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"}) + err = operations.Check(ctx, &operations.CheckOpt{Fsrc: r.Flocal, Fdst: r.Fremote}) // should not error even though dst has transformed names + assert.NoError(t, err) +} + +func TestRunTwice(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "dir,prefix=tac", "dir,prefix=tic") + require.NoError(t, err) + file1 := r.WriteFile("toe/toe/toe.txt", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1)) + + // result should not change second time, since src is unchanged + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1)) +} + +func TestSyntax(t *testing.T) { + ctx := context.Background() + err := transform.SetOptions(ctx, "prefix") + assert.Error(t, err) // should error as required value is missing + + err = transform.SetOptions(ctx, "banana") + assert.Error(t, err) // should error as unrecognized option + + err = transform.SetOptions(ctx, "=123") + assert.Error(t, err) // should error as required key is missing + + err = transform.SetOptions(ctx, "prefix=123") + assert.NoError(t, err) // should not error +} + +func TestConflicting(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "prefix=tac", "trimprefix=tac") + require.NoError(t, err) + file1 := r.WriteFile("toe/toe/toe", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + // should result in no change as prefix and trimprefix cancel out + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("toe/toe/toe", "hello world", t1)) +} + +func TestMove(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic") + require.NoError(t, err) + r.WriteFile("toe/toe/toe.txt", "hello world", t1) + _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1) + require.NoError(t, err) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = MoveDir(ctx, r.Fremote, r.Flocal, true, true) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalListing(t, []fstest.Item{}, []string{}) + r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"}) +} + +func TestTransformFile(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic") + require.NoError(t, err) + r.WriteFile("toe/toe/toe.txt", "hello world", t1) + _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1) + require.NoError(t, err) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = MoveDir(ctx, r.Fremote, r.Flocal, true, true) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalListing(t, []fstest.Item{}, []string{}) + r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"}) + + err = transform.SetOptions(ctx, "all,trimprefix=tic", "all,trimprefix=tac") + require.NoError(t, err) + err = operations.TransformFile(ctx, r.Fremote, "tictactoe/tictactoe/tictactoe.txt") + require.NoError(t, err) + r.CheckLocalListing(t, []fstest.Item{}, []string{}) + r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe", "toe", "toe/toe"}) +} + +func TestManualTransformFile(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + + r.Flocal.Features().DisableList([]string{"Copy", "Move"}) + r.Fremote.Features().DisableList([]string{"Copy", "Move"}) + + err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic") + require.NoError(t, err) + r.WriteFile("toe/toe/toe.txt", "hello world", t1) + _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1) + require.NoError(t, err) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = MoveDir(ctx, r.Fremote, r.Flocal, true, true) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalListing(t, []fstest.Item{}, []string{}) + r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"}) + + err = transform.SetOptions(ctx, "all,trimprefix=tic", "all,trimprefix=tac") + require.NoError(t, err) + err = operations.TransformFile(ctx, r.Fremote, "tictactoe/tictactoe/tictactoe.txt") + require.NoError(t, err) + r.CheckLocalListing(t, []fstest.Item{}, []string{}) + r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe", "toe", "toe/toe"}) +} + +func TestBase64(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "all,base64encode") + require.NoError(t, err) + file1 := r.WriteFile("toe/toe/toe.txt", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("dG9l/dG9l/dG9lLnR4dA==", "hello world", t1)) + + // round trip + err = transform.SetOptions(ctx, "all,base64decode") + require.NoError(t, err) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Flocal, r.Fremote, true) + testLoggerVsLsf(ctx, r.Flocal, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("dG9l/dG9l/dG9lLnR4dA==", "hello world", t1)) +} + +func TestError(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "all,prefix=ta/c") // has illegal character + require.NoError(t, err) + file1 := r.WriteFile("toe/toe/toe", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + // ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + // testLoggerVsLsf(ctx, r.Fremote, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + assert.Error(t, err) + accounting.GlobalStats().ResetCounters() + + r.CheckLocalListing(t, []fstest.Item{file1}, []string{"toe", "toe/toe"}) + r.CheckRemoteListing(t, []fstest.Item{file1}, []string{"toe", "toe/toe"}) +} diff --git a/magefile.go b/magefile.go index be97194..caca199 100644 --- a/magefile.go +++ b/magefile.go @@ -116,6 +116,21 @@ func TestAgainstContainer() error { if err != nil { return err } + err = runCommandWithEnv( + []string{"RCLONE_CONFIG=" + config.ConfigPath}, + "go", "-C", "fs/sync", "test", "-parallel=1", "-remote", "TestStudIP:fs/sync", "-v", "-count=1", + ) + if err != nil { + return err + } + + err = runCommandWithEnv( + []string{"RCLONE_CONFIG=" + config.ConfigPath}, + "go", "-C", "fs/sync", "test", "-parallel=1", "-remote", "TestStudIP:fs/operations", "-v", "-count=1", + ) + if err != nil { + return err + } return nil } |