diff options
Diffstat (limited to 'fs/operations/operations.go')
| -rw-r--r-- | fs/operations/operations.go | 2742 |
1 files changed, 2742 insertions, 0 deletions
diff --git a/fs/operations/operations.go b/fs/operations/operations.go new file mode 100644 index 0000000..1ada8f8 --- /dev/null +++ b/fs/operations/operations.go @@ -0,0 +1,2742 @@ +// Package operations does generic operations on filesystems and objects +package operations + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/csv" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "os" + "path" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/object" + "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/lib/atexit" + "github.com/rclone/rclone/lib/errcount" + "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/random" + "github.com/rclone/rclone/lib/readers" + "github.com/rclone/rclone/lib/transform" + "golang.org/x/sync/errgroup" + "golang.org/x/text/unicode/norm" +) + +// CheckHashes checks the two files to see if they have common +// known hash types and compares them +// +// Returns. +// +// equal - which is equality of the hashes +// +// hash - the HashType. This is HashNone if either of the hashes were +// unset or a compatible hash couldn't be found. +// +// err - may return an error which will already have been logged +// +// If an error is returned it will return equal as false +func CheckHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object) (equal bool, ht hash.Type, err error) { + common := src.Fs().Hashes().Overlap(dst.Fs().Hashes()) + // fs.Debugf(nil, "Shared hashes: %v", common) + if common.Count() == 0 { + return true, hash.None, nil + } + equal, ht, _, _, err = checkHashes(ctx, src, dst, common.GetOne()) + return equal, ht, err +} + +var errNoHash = errors.New("no hash available") + +// checkHashes does the work of CheckHashes but takes a hash.Type and +// returns the effective hash type used. +func checkHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object, ht hash.Type) (equal bool, htOut hash.Type, srcHash, dstHash string, err error) { + // Calculate hashes in parallel + g, ctx := errgroup.WithContext(ctx) + var srcErr, dstErr error + g.Go(func() (err error) { + srcHash, srcErr = src.Hash(ctx, ht) + if srcErr != nil { + return srcErr + } + if srcHash == "" { + fs.Debugf(src, "Src hash empty - aborting Dst hash check") + return errNoHash + } + return nil + }) + g.Go(func() (err error) { + dstHash, dstErr = dst.Hash(ctx, ht) + if dstErr != nil { + return dstErr + } + if dstHash == "" { + fs.Debugf(dst, "Dst hash empty - aborting Src hash check") + return errNoHash + } + return nil + }) + err = g.Wait() + if err == errNoHash { + return true, hash.None, srcHash, dstHash, nil + } + if srcErr != nil { + err = fs.CountError(ctx, srcErr) + fs.Errorf(src, "Failed to calculate src hash: %v", err) + } + if dstErr != nil { + err = fs.CountError(ctx, dstErr) + fs.Errorf(dst, "Failed to calculate dst hash: %v", err) + } + if err != nil { + return false, ht, srcHash, dstHash, err + } + if srcHash != dstHash { + fs.Debugf(src, "%v = %s (%v)", ht, srcHash, src.Fs()) + fs.Debugf(dst, "%v = %s (%v)", ht, dstHash, dst.Fs()) + return false, ht, srcHash, dstHash, nil + } + fs.Debugf(src, "%v = %s OK", ht, srcHash) + return true, ht, srcHash, dstHash, nil +} + +// Equal checks to see if the src and dst objects are equal by looking at +// size, mtime and hash +// +// If the src and dst size are different then it is considered to be +// not equal. If --size-only is in effect then this is the only check +// that is done. If --ignore-size is in effect then this check is +// skipped and the files are considered the same size. +// +// If the size is the same and the mtime is the same then it is +// considered to be equal. This check is skipped if using --checksum. +// +// If the size is the same and mtime is different, unreadable or +// --checksum is set and the hash is the same then the file is +// considered to be equal. In this case the mtime on the dst is +// updated if --checksum is not set. +// +// Otherwise the file is considered to be not equal including if there +// were errors reading info. +func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool { + return equal(ctx, src, dst, defaultEqualOpt(ctx)) +} + +// DirsEqual is like Equal but for dirs instead of objects. +// It returns true if two dirs should be considered "equal" for the purposes of syncCopyMove +// (in other words, true == "skip updating modtime/metadata for this dir".) +// Unlike Equal, it does not consider size or checksum, as these do not apply to directories. +func DirsEqual(ctx context.Context, src, dst fs.Directory, opt DirsEqualOpt) (equal bool) { + if dst == nil { + return false + } + ci := fs.GetConfig(ctx) + if ci.SizeOnly || ci.Immutable || ci.IgnoreExisting || opt.ModifyWindow == fs.ModTimeNotSupported { + return true + } + if ci.IgnoreTimes { + return false + } + if !(opt.SetDirModtime || opt.SetDirMetadata) { + return true + } + srcModTime, dstModTime := src.ModTime(ctx), dst.ModTime(ctx) + if srcModTime.IsZero() || dstModTime.IsZero() { + return false + } + dt := dstModTime.Sub(srcModTime) + if dt < opt.ModifyWindow && dt > -opt.ModifyWindow { + fs.Debugf(dst, "Directory modification time the same (differ by %s, within tolerance %s)", dt, opt.ModifyWindow) + return true + } + if ci.UpdateOlder && dt >= opt.ModifyWindow { + fs.Debugf(dst, "Destination directory is newer than source, skipping") + return true + } + return false +} + +// sizeDiffers compare the size of src and dst taking into account the +// various ways of ignoring sizes +func sizeDiffers(ctx context.Context, src, dst fs.ObjectInfo) bool { + ci := fs.GetConfig(ctx) + if ci.IgnoreSize || src.Size() < 0 || dst.Size() < 0 { + return false + } + if src.Size() == dst.Size() { + fs.Debugf(dst, "size = %d OK", dst.Size()) + return false + } + fs.Debugf(src, "size = %d (%v)", src.Size(), src.Fs()) + fs.Debugf(dst, "size = %d (%v)", dst.Size(), dst.Fs()) + return true +} + +var checksumWarning sync.Once + +// options for equal function() +type equalOpt struct { + sizeOnly bool // if set only check size + checkSum bool // if set check checksum+size instead of modtime+size + updateModTime bool // if set update the modtime if hashes identical and checking with modtime+size + forceModTimeMatch bool // if set assume modtimes match +} + +// default set of options for equal() +func defaultEqualOpt(ctx context.Context) equalOpt { + ci := fs.GetConfig(ctx) + return equalOpt{ + sizeOnly: ci.SizeOnly, + checkSum: ci.CheckSum, + updateModTime: !ci.NoUpdateModTime, + forceModTimeMatch: false, + } +} + +// DirsEqualOpt represents options for DirsEqual function() +type DirsEqualOpt struct { + ModifyWindow time.Duration // Max time diff to be considered the same + SetDirModtime bool // whether to consider dir modtime + SetDirMetadata bool // whether to consider dir metadata +} + +var modTimeUploadOnce sync.Once + +// emit a log if we are about to upload a file to set its modification time +func logModTimeUpload(dst fs.Object) { + modTimeUploadOnce.Do(func() { + fs.Logf(dst.Fs(), "Forced to upload files to set modification times on this backend.") + }) +} + +// EqualFn allows replacing Equal() with a custom function during NeedTransfer() +type ( + EqualFn func(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool + equalFnContextKey struct{} +) + +var equalFnKey = equalFnContextKey{} + +// WithEqualFn stores equalFn in ctx and returns a copy of ctx in which equalFnKey = equalFn +func WithEqualFn(ctx context.Context, equalFn EqualFn) context.Context { + return context.WithValue(ctx, equalFnKey, equalFn) +} + +func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt) bool { + ci := fs.GetConfig(ctx) + logger, _ := GetLogger(ctx) + if sizeDiffers(ctx, src, dst) { + fs.Debug(src, "Sizes differ") + logger(ctx, Differ, src, dst, nil) + return false + } + if opt.sizeOnly { + fs.Debugf(src, "Sizes identical") + logger(ctx, Match, src, dst, nil) + return true + } + + // Assert: Size is equal or being ignored + + // If checking checksum and not modtime + if opt.checkSum { + // Check the hash + same, ht, _ := CheckHashes(ctx, src, dst) + if !same { + fs.Debugf(src, "%v differ", ht) + logger(ctx, Differ, src, dst, nil) + return false + } + if ht == hash.None { + common := src.Fs().Hashes().Overlap(dst.Fs().Hashes()) + if common.Count() == 0 { + checksumWarning.Do(func() { + fs.Logf(dst.Fs(), "--checksum is in use but the source and destination have no hashes in common; falling back to --size-only") + }) + } + fs.Debugf(src, "Size of src and dst objects identical") + } else { + fs.Debugf(src, "Size and %v of src and dst objects identical", ht) + } + logger(ctx, Match, src, dst, nil) + return true + } + + srcModTime := src.ModTime(ctx) + if !opt.forceModTimeMatch { + // Sizes the same so check the mtime + modifyWindow := fs.GetModifyWindow(ctx, src.Fs(), dst.Fs()) + if modifyWindow == fs.ModTimeNotSupported { + fs.Debugf(src, "Sizes identical") + logger(ctx, Match, src, dst, nil) + return true + } + dstModTime := dst.ModTime(ctx) + dt := dstModTime.Sub(srcModTime) + if dt < modifyWindow && dt > -modifyWindow { + fs.Debugf(src, "Size and modification time the same (differ by %s, within tolerance %s)", dt, modifyWindow) + logger(ctx, Match, src, dst, nil) + return true + } + + fs.Debugf(src, "Modification times differ by %s: %v, %v", dt, srcModTime, dstModTime) + } + + // Check if the hashes are the same + same, ht, _ := CheckHashes(ctx, src, dst) + if !same { + fs.Debugf(src, "%v differ", ht) + logger(ctx, Differ, src, dst, nil) + return false + } + if ht == hash.None && !ci.RefreshTimes { + // if couldn't check hash, return that they differ + logger(ctx, Differ, src, dst, nil) + return false + } + + // mod time differs but hash is the same to reset mod time if required + if opt.updateModTime { + if !SkipDestructive(ctx, src, "update modification time") { + // Size and hash the same but mtime different + // Error if objects are treated as immutable + if ci.Immutable { + fs.Errorf(dst, "Timestamp mismatch between immutable objects") + logger(ctx, Differ, src, dst, nil) + return false + } + // Update the mtime of the dst object here + err := dst.SetModTime(ctx, srcModTime) + if errors.Is(err, fs.ErrorCantSetModTime) { + logModTimeUpload(dst) + fs.Infof(dst, "src and dst identical but can't set mod time without re-uploading") + logger(ctx, Differ, src, dst, nil) + return false + } else if errors.Is(err, fs.ErrorCantSetModTimeWithoutDelete) { + logModTimeUpload(dst) + fs.Infof(dst, "src and dst identical but can't set mod time without deleting and re-uploading") + // Remove the file if BackupDir isn't set. If BackupDir is set we would rather have the old file + // put in the BackupDir than deleted which is what will happen if we don't delete it. + if ci.BackupDir == "" { + err = dst.Remove(ctx) + if err != nil { + fs.Errorf(dst, "failed to delete before re-upload: %v", err) + } + } + logger(ctx, Differ, src, dst, nil) + return false + } else if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(dst, "Failed to set modification time: %v", err) + } else { + fs.Infof(src, "Updated modification time in destination") + } + } + } + logger(ctx, Match, src, dst, nil) + return true +} + +// CommonHash returns a single hash.Type and a HashOption with that +// type which is in common between the two fs.Fs. +func CommonHash(ctx context.Context, fa, fb fs.Info) (hash.Type, *fs.HashesOption) { + ci := fs.GetConfig(ctx) + // work out which hash to use - limit to 1 hash in common + var common hash.Set + hashType := hash.None + if !ci.IgnoreChecksum { + common = fb.Hashes().Overlap(fa.Hashes()) + if common.Count() > 0 { + hashType = common.GetOne() + common = hash.Set(hashType) + } + } + return hashType, &fs.HashesOption{Hashes: common} +} + +// SameObject returns true if src and dst could be pointing to the +// same object. +func SameObject(src, dst fs.Object) bool { + srcFs, dstFs := src.Fs(), dst.Fs() + if !SameConfig(srcFs, dstFs) { + // If same remote type then check ID of objects if available + doSrcID, srcIDOK := src.(fs.IDer) + doDstID, dstIDOK := dst.(fs.IDer) + if srcIDOK && dstIDOK && SameRemoteType(srcFs, dstFs) { + srcID, dstID := doSrcID.ID(), doDstID.ID() + if srcID != "" && srcID == dstID { + return true + } + } + return false + } + srcPath := path.Join(srcFs.Root(), src.Remote()) + dstPath := path.Join(dstFs.Root(), dst.Remote()) + if srcFs.Features().IsLocal && dstFs.Features().IsLocal && runtime.GOOS == "darwin" { + if norm.NFC.String(srcPath) == norm.NFC.String(dstPath) { + return true + } + } + if dst.Fs().Features().CaseInsensitive { + srcPath = strings.ToLower(srcPath) + dstPath = strings.ToLower(dstPath) + } + return srcPath == dstPath +} + +// Move src object to dst or fdst if nil. If dst is nil then it uses +// remote as the name of the new object. +// +// Note that you must check the destination does not exist before +// calling this and pass it as dst. If you pass dst=nil and the +// destination does exist then this may create duplicates or return +// errors. +// +// It returns the destination object if possible. Note that this may +// be nil. +// +// This is accounted as a check. +func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) { + return move(ctx, fdst, dst, remote, src, false) +} + +// MoveTransfer moves src object to dst or fdst if nil. If dst is nil +// then it uses remote as the name of the new object. +// +// This is identical to Move but is accounted as a transfer. +func MoveTransfer(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) { + return move(ctx, fdst, dst, remote, src, true) +} + +// move - see Move for help +func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object, isTransfer bool) (newDst fs.Object, err error) { + origRemote := remote // avoid double-transform on fallback to copy + remote = transform.Path(ctx, remote, false) + ci := fs.GetConfig(ctx) + newDst = dst + if ci.DryRun && dst != nil && SameObject(src, dst) && src.Remote() == transform.Path(ctx, dst.Remote(), false) { + return // avoid SkipDestructive log for objects that won't really be moved + } + var tr *accounting.Transfer + if isTransfer { + tr = accounting.Stats(ctx).NewTransfer(src, fdst) + } else { + tr = accounting.Stats(ctx).NewCheckingTransfer(src, "moving") + } + defer func() { + if err == nil { + accounting.Stats(ctx).Renames(1) + } + tr.Done(ctx, err) + }() + action := "move" + if remote != src.Remote() { + action += " to " + remote + } + if SkipDestructive(ctx, src, action) { + in := tr.Account(ctx, nil) + in.DryRun(src.Size()) + return newDst, nil + } + // See if we have Move available + if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && (fdst.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) { + // Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive) + if dst != nil { + remote = transform.Path(ctx, dst.Remote(), false) + if !SameObject(src, dst) { + err = DeleteFile(ctx, dst) + if err != nil { + return newDst, err + } + } else if src.Remote() == remote { + return newDst, nil + } else if needsMoveCaseInsensitive(fdst, fdst, remote, src.Remote(), false) { + doMove = func(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + return MoveCaseInsensitive(ctx, fdst, fdst, remote, src.Remote(), false, src) + } + } + } else if needsMoveCaseInsensitive(fdst, fdst, remote, src.Remote(), false) { + doMove = func(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + return MoveCaseInsensitive(ctx, fdst, fdst, remote, src.Remote(), false, src) + } + } + // Move dst <- src + in := tr.Account(ctx, nil) // account the transfer + in.ServerSideTransferStart() + newDst, err = doMove(ctx, src, remote) + switch err { + case nil: + if newDst != nil && src.String() != newDst.String() { + fs.Infof(src, "Moved (server-side) to: %s", newDst.String()) + } else { + fs.Infof(src, "Moved (server-side)") + } + in.ServerSideMoveEnd(newDst.Size()) // account the bytes for the server-side transfer + _ = in.Close() + return newDst, nil + case fs.ErrorCantMove: + fs.Debugf(src, "Can't move, switching to copy") + _ = in.Close() + default: + err = fs.CountError(ctx, err) + fs.Errorf(src, "Couldn't move: %v", err) + _ = in.Close() + return newDst, err + } + } + // Move not found or didn't work so copy dst <- src + if origRemote != remote { + dst = nil + } + newDst, err = Copy(ctx, fdst, dst, origRemote, src) + if err != nil { + fs.Errorf(src, "Not deleting source as copy failed: %v", err) + return newDst, err + } + // Delete src if no error on copy + return newDst, DeleteFile(ctx, src) +} + +// CanServerSideMove returns true if fdst support server-side moves or +// server-side copies +// +// Some remotes simulate rename by server-side copy and delete, so include +// remotes that implements either Mover or Copier. +func CanServerSideMove(fdst fs.Fs) bool { + canMove := fdst.Features().Move != nil + canCopy := fdst.Features().Copy != nil + return canMove || canCopy +} + +// SuffixName adds the current --suffix to the remote, obeying +// --suffix-keep-extension if set +func SuffixName(ctx context.Context, remote string) string { + ci := fs.GetConfig(ctx) + if ci.Suffix == "" { + return remote + } + if ci.SuffixKeepExtension { + return transform.SuffixKeepExtension(remote, ci.Suffix) + } + return remote + ci.Suffix +} + +// DeleteFileWithBackupDir deletes a single file respecting --dry-run +// and accumulating stats and errors. +// +// If backupDir is set then it moves the file to there instead of +// deleting +func DeleteFileWithBackupDir(ctx context.Context, dst fs.Object, backupDir fs.Fs) (err error) { + tr := accounting.Stats(ctx).NewCheckingTransfer(dst, "deleting") + defer func() { + tr.Done(ctx, err) + }() + err = accounting.Stats(ctx).DeleteFile(ctx, dst.Size()) + if err != nil { + return err + } + action, actioned := "delete", "Deleted" + if backupDir != nil { + action, actioned = "move into backup dir", "Moved into backup dir" + } + skip := SkipDestructive(ctx, dst, action) + if skip { + // do nothing + } else if backupDir != nil { + err = MoveBackupDir(ctx, backupDir, dst) + } else { + err = dst.Remove(ctx) + } + if err != nil { + fs.Errorf(dst, "Couldn't %s: %v", action, err) + err = fs.CountError(ctx, err) + } else if !skip { + fs.Infof(dst, "%s", actioned) + } + return err +} + +// DeleteFile deletes a single file respecting --dry-run and accumulating stats and errors. +// +// If useBackupDir is set and --backup-dir is in effect then it moves +// the file to there instead of deleting +func DeleteFile(ctx context.Context, dst fs.Object) (err error) { + return DeleteFileWithBackupDir(ctx, dst, nil) +} + +// DeleteFilesWithBackupDir removes all the files passed in the +// channel +// +// If backupDir is set the files will be placed into that directory +// instead of being deleted. +func DeleteFilesWithBackupDir(ctx context.Context, toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error { + var wg sync.WaitGroup + ci := fs.GetConfig(ctx) + wg.Add(ci.Checkers) + var errorCount atomic.Int32 + var fatalErrorCount atomic.Int32 + + for range ci.Checkers { + go func() { + defer wg.Done() + for dst := range toBeDeleted { + err := DeleteFileWithBackupDir(ctx, dst, backupDir) + if err != nil { + errorCount.Add(1) + logger, _ := GetLogger(ctx) + logger(ctx, TransferError, nil, dst, err) + if fserrors.IsFatalError(err) { + fs.Errorf(dst, "Got fatal error on delete: %s", err) + fatalErrorCount.Add(1) + return + } + } + } + }() + } + fs.Debugf(nil, "Waiting for deletions to finish") + wg.Wait() + if errorCount.Load() > 0 { + err := fmt.Errorf("failed to delete %d files", errorCount.Load()) + if fatalErrorCount.Load() > 0 { + return fserrors.FatalError(err) + } + return err + } + return nil +} + +// DeleteFiles removes all the files passed in the channel +func DeleteFiles(ctx context.Context, toBeDeleted fs.ObjectsChan) error { + return DeleteFilesWithBackupDir(ctx, toBeDeleted, nil) +} + +// ReadFile reads the object into memory and accounts it +func ReadFile(ctx context.Context, o fs.Object) (b []byte, err error) { + tr := accounting.Stats(ctx).NewTransfer(o, nil) + defer func() { + tr.Done(ctx, err) + }() + in0, err := Open(ctx, o) + if err != nil { + return nil, fmt.Errorf("failed to open %v: %w", o, err) + } + in := tr.Account(ctx, in0).WithBuffer() // account and buffer the transfer + defer fs.CheckClose(in, &err) // closes in0 also + b, err = io.ReadAll(in) + if err != nil { + return nil, fmt.Errorf("failed to read %v: %w", o, err) + } + return b, nil +} + +// SameRemoteType returns true if fdst and fsrc are the same type +func SameRemoteType(fdst, fsrc fs.Info) bool { + return fmt.Sprintf("%T", fdst) == fmt.Sprintf("%T", fsrc) +} + +// SameConfig returns true if fdst and fsrc are using the same config +// file entry +func SameConfig(fdst, fsrc fs.Info) bool { + return fdst.Name() == fsrc.Name() +} + +// SameConfigArr returns true if any of []fsrcs has same config file entry with fdst +func SameConfigArr(fdst fs.Info, fsrcs []fs.Fs) bool { + for _, fsrc := range fsrcs { + if fdst.Name() == fsrc.Name() { + return true + } + } + return false +} + +// Same returns true if fdst and fsrc point to the same underlying Fs +func Same(fdst, fsrc fs.Info) bool { + return SameConfig(fdst, fsrc) && strings.Trim(fdst.Root(), "/") == strings.Trim(fsrc.Root(), "/") +} + +// fixRoot returns the Root with a trailing / if not empty. +// +// It returns a case folded version for case insensitive file systems +func fixRoot(f fs.Info) (s string, folded string) { + s = strings.Trim(filepath.ToSlash(f.Root()), "/") + if s != "" { + s += "/" + } + folded = s + if f.Features().CaseInsensitive { + folded = strings.ToLower(s) + } + return s, folded +} + +// OverlappingFilterCheck returns true if fdst and fsrc point to the same +// underlying Fs and they overlap without fdst being excluded by any filter rule. +func OverlappingFilterCheck(ctx context.Context, fdst fs.Fs, fsrc fs.Fs) bool { + if !SameConfig(fdst, fsrc) { + return false + } + fdstRoot, fdstRootFolded := fixRoot(fdst) + fsrcRoot, fsrcRootFolded := fixRoot(fsrc) + if fdstRootFolded == fsrcRootFolded { + return true + } else if strings.HasPrefix(fdstRootFolded, fsrcRootFolded) { + fdstRelative := fdstRoot[len(fsrcRoot):] + return filterCheck(ctx, fsrc, fdstRelative) + } else if strings.HasPrefix(fsrcRootFolded, fdstRootFolded) { + fsrcRelative := fsrcRoot[len(fdstRoot):] + return filterCheck(ctx, fdst, fsrcRelative) + } + return false +} + +// filterCheck checks if dir is included in f +func filterCheck(ctx context.Context, f fs.Fs, dir string) bool { + fi := filter.GetConfig(ctx) + includeDirectory := fi.IncludeDirectory(ctx, f) + include, err := includeDirectory(dir) + if err != nil { + fs.Errorf(f, "Failed to discover whether directory is included: %v", err) + return true + } + return include +} + +// SameDir returns true if fdst and fsrc point to the same +// underlying Fs and they are the same directory. +func SameDir(fdst, fsrc fs.Info) bool { + if !SameConfig(fdst, fsrc) { + return false + } + _, fdstRootFolded := fixRoot(fdst) + _, fsrcRootFolded := fixRoot(fsrc) + return fdstRootFolded == fsrcRootFolded +} + +// Retry runs fn up to maxTries times if it returns a retriable error +func Retry(ctx context.Context, o any, maxTries int, fn func() error) (err error) { + for tries := 1; tries <= maxTries; tries++ { + // Call the function which might error + err = fn() + if err == nil { + break + } + // End if ctx is in error + if fserrors.ContextError(ctx, &err) { + break + } + // Retry if err returned a retry error + if fserrors.IsRetryError(err) || fserrors.ShouldRetry(err) { + fs.Debugf(o, "Received error: %v - low level retry %d/%d", err, tries, maxTries) + continue + } else if t, ok := pacer.IsRetryAfter(err); ok { + fs.Debugf(o, "Sleeping for %v (as indicated by the server) to obey Retry-After error: %v", t, err) + time.Sleep(t) + continue + } + break + } + return err +} + +// ListFn lists the Fs to the supplied function +// +// Lists in parallel which may get them out of order +func ListFn(ctx context.Context, f fs.Fs, fn func(fs.Object)) error { + ci := fs.GetConfig(ctx) + return walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error { + entries.ForObject(fn) + return nil + }) +} + +// StdoutMutex mutex for synchronized output on stdout +var StdoutMutex sync.Mutex + +// SyncPrintf is a global var holding the Printf function so that it +// can be overridden. +// +// This writes to stdout holding the StdoutMutex. If you are going to +// override it and write to os.Stdout then you should hold the +// StdoutMutex too. +var SyncPrintf = func(format string, a ...any) { + StdoutMutex.Lock() + defer StdoutMutex.Unlock() + fmt.Printf(format, a...) +} + +// SyncFprintf - Synchronized fmt.Fprintf +// +// Ignores errors from Fprintf. +// +// Prints to stdout if w is nil +func SyncFprintf(w io.Writer, format string, a ...any) { + if w == nil || w == os.Stdout { + SyncPrintf(format, a...) + } else { + StdoutMutex.Lock() + defer StdoutMutex.Unlock() + _, _ = fmt.Fprintf(w, format, a...) + } +} + +// SizeString make string representation of size for output +// +// Optional human-readable format including a binary suffix +func SizeString(size int64, humanReadable bool) string { + if humanReadable { + if size < 0 { + return "-" + fs.SizeSuffix(-size).String() + } + return fs.SizeSuffix(size).String() + } + return strconv.FormatInt(size, 10) +} + +// SizeStringField make string representation of size for output in fixed width field +// +// Optional human-readable format including a binary suffix +// Argument rawWidth is used to format field with of raw value. When humanReadable +// option the width is hard coded to 9, since SizeSuffix strings have precision 3 +// and longest value will be "999.999Ei". This way the width can be optimized +// depending to the humanReadable option. To always use a longer width the return +// value can always be fed into another format string with a specific field with. +func SizeStringField(size int64, humanReadable bool, rawWidth int) string { + str := SizeString(size, humanReadable) + if humanReadable { + return fmt.Sprintf("%9s", str) + } + return fmt.Sprintf("%[2]*[1]s", str, rawWidth) +} + +// CountString make string representation of count for output +// +// Optional human-readable format including a decimal suffix +func CountString(count int64, humanReadable bool) string { + if humanReadable { + if count < 0 { + return "-" + fs.CountSuffix(-count).String() + } + return fs.CountSuffix(count).String() + } + return strconv.FormatInt(count, 10) +} + +// CountStringField make string representation of count for output in fixed width field +// +// Similar to SizeStringField, but human readable with decimal prefix and field width 8 +// since there is no 'i' in the decimal prefix symbols (e.g. "999.999E") +func CountStringField(count int64, humanReadable bool, rawWidth int) string { + str := CountString(count, humanReadable) + if humanReadable { + return fmt.Sprintf("%8s", str) + } + return fmt.Sprintf("%[2]*[1]s", str, rawWidth) +} + +// List the Fs to the supplied writer +// +// Shows size and path - obeys includes and excludes. +// +// Lists in parallel which may get them out of order +func List(ctx context.Context, f fs.Fs, w io.Writer) error { + ci := fs.GetConfig(ctx) + return ListFn(ctx, f, func(o fs.Object) { + SyncFprintf(w, "%s %s\n", SizeStringField(o.Size(), ci.HumanReadable, 9), o.Remote()) + }) +} + +// ListLong lists the Fs to the supplied writer +// +// Shows size, mod time and path - obeys includes and excludes. +// +// Lists in parallel which may get them out of order +func ListLong(ctx context.Context, f fs.Fs, w io.Writer) error { + ci := fs.GetConfig(ctx) + return ListFn(ctx, f, func(o fs.Object) { + tr := accounting.Stats(ctx).NewCheckingTransfer(o, "listing") + defer func() { + tr.Done(ctx, nil) + }() + modTime := o.ModTime(ctx) + SyncFprintf(w, "%s %s %s\n", SizeStringField(o.Size(), ci.HumanReadable, 9), modTime.Local().Format("2006-01-02 15:04:05.000000000"), o.Remote()) + }) +} + +// HashSum returns the human-readable hash for ht passed in. This may +// be UNSUPPORTED or ERROR. If it isn't returning a valid hash it will +// return an error. +func HashSum(ctx context.Context, ht hash.Type, base64Encoded bool, downloadFlag bool, o fs.Object) (string, error) { + var sum string + var err error + + // If downloadFlag is true, download and hash the file. + // If downloadFlag is false, call o.Hash asking the remote for the hash + if downloadFlag { + // Setup: Define accounting, open the file with NewReOpen to provide restarts, account for the transfer, and setup a multi-hasher with the appropriate type + // Execution: io.Copy file to hasher, get hash and encode in hex + + tr := accounting.Stats(ctx).NewTransfer(o, nil) + defer func() { + tr.Done(ctx, err) + }() + + // Open with NewReOpen to provide restarts + var options []fs.OpenOption + for _, option := range fs.GetConfig(ctx).DownloadHeaders { + options = append(options, option) + } + var in io.ReadCloser + in, err = Open(ctx, o, options...) + if err != nil { + return "ERROR", fmt.Errorf("failed to open file %v: %w", o, err) + } + + // Account and buffer the transfer + in = tr.Account(ctx, in).WithBuffer() + + // Setup hasher + hasher, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht)) + if err != nil { + return "UNSUPPORTED", fmt.Errorf("hash unsupported: %w", err) + } + + // Copy to hasher, downloading the file and passing directly to hash + _, err = io.Copy(hasher, in) + if err != nil { + return "ERROR", fmt.Errorf("failed to copy file to hasher: %w", err) + } + + // Get hash as hex or base64 encoded string + sum, err = hasher.SumString(ht, base64Encoded) + if err != nil { + return "ERROR", fmt.Errorf("hasher returned an error: %w", err) + } + } else { + tr := accounting.Stats(ctx).NewCheckingTransfer(o, "hashing") + defer func() { + tr.Done(ctx, err) + }() + + sum, err = o.Hash(ctx, ht) + if base64Encoded { + hexBytes, _ := hex.DecodeString(sum) + sum = base64.URLEncoding.EncodeToString(hexBytes) + } + if err == hash.ErrUnsupported { + return "", fmt.Errorf("hash unsupported: %w", err) + } + if err != nil { + return "", fmt.Errorf("failed to get hash %v from backend: %w", ht, err) + } + } + + return sum, nil +} + +// HashLister does an md5sum equivalent for the hash type passed in +// Updated to handle both standard hex encoding and base64 +// Updated to perform multiple hashes concurrently +func HashLister(ctx context.Context, ht hash.Type, outputBase64 bool, downloadFlag bool, f fs.Fs, w io.Writer) error { + width := hash.Width(ht, outputBase64) + // Use --checkers concurrency unless downloading in which case use --transfers + concurrency := fs.GetConfig(ctx).Checkers + if downloadFlag { + concurrency = fs.GetConfig(ctx).Transfers + } + concurrencyControl := make(chan struct{}, concurrency) + var wg sync.WaitGroup + err := ListFn(ctx, f, func(o fs.Object) { + wg.Add(1) + concurrencyControl <- struct{}{} + go func() { + defer func() { + <-concurrencyControl + wg.Done() + }() + sum, err := HashSum(ctx, ht, outputBase64, downloadFlag, o) + if err != nil { + fs.Errorf(o, "%v", fs.CountError(ctx, err)) + return + } + SyncFprintf(w, "%*s %s\n", width, sum, o.Remote()) + }() + }) + wg.Wait() + return err +} + +// HashSumStream outputs a line compatible with md5sum to w based on the +// input stream in and the hash type ht passed in. If outputBase64 is +// set then the hash will be base64 instead of hexadecimal. +func HashSumStream(ht hash.Type, outputBase64 bool, in io.ReadCloser, w io.Writer) error { + hasher, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht)) + if err != nil { + return fmt.Errorf("hash unsupported: %w", err) + } + written, err := io.Copy(hasher, in) + fs.Debugf(nil, "Creating %s hash of %d bytes read from input stream", ht, written) + if err != nil { + return fmt.Errorf("failed to copy input to hasher: %w", err) + } + sum, err := hasher.SumString(ht, outputBase64) + if err != nil { + return fmt.Errorf("hasher returned an error: %w", err) + } + width := hash.Width(ht, outputBase64) + SyncFprintf(w, "%*s -\n", width, sum) + return nil +} + +// Count counts the objects and their sizes in the Fs +// +// Obeys includes and excludes +func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, sizelessObjects int64, err error) { + err = ListFn(ctx, f, func(o fs.Object) { + atomic.AddInt64(&objects, 1) + objectSize := o.Size() + if objectSize < 0 { + atomic.AddInt64(&sizelessObjects, 1) + } else if objectSize > 0 { + atomic.AddInt64(&size, objectSize) + } + }) + return +} + +// ConfigMaxDepth returns the depth to use for a recursive or non recursive listing. +func ConfigMaxDepth(ctx context.Context, recursive bool) int { + ci := fs.GetConfig(ctx) + depth := ci.MaxDepth + if !recursive && depth < 0 { + depth = 1 + } + return depth +} + +// ListDir lists the directories/buckets/containers in the Fs to the supplied writer +func ListDir(ctx context.Context, f fs.Fs, w io.Writer) error { + ci := fs.GetConfig(ctx) + return walk.ListR(ctx, f, "", false, ConfigMaxDepth(ctx, false), walk.ListDirs, func(entries fs.DirEntries) error { + entries.ForDir(func(dir fs.Directory) { + if dir != nil { + SyncFprintf(w, "%s %13s %s %s\n", SizeStringField(dir.Size(), ci.HumanReadable, 12), dir.ModTime(ctx).Local().Format("2006-01-02 15:04:05"), CountStringField(dir.Items(), ci.HumanReadable, 9), dir.Remote()) + } + }) + return nil + }) +} + +// Mkdir makes a destination directory or container +func Mkdir(ctx context.Context, f fs.Fs, dir string) error { + if SkipDestructive(ctx, fs.LogDirName(f, dir), "make directory") { + return nil + } + fs.Infof(fs.LogDirName(f, dir), "Making directory") + err := f.Mkdir(ctx, dir) + if err != nil { + err = fs.CountError(ctx, err) + return err + } + return nil +} + +// MkdirMetadata makes a destination directory or container with metadata +// +// If the destination Fs doesn't support this it will fall back to +// Mkdir and in this case newDst will be nil. +func MkdirMetadata(ctx context.Context, f fs.Fs, dir string, metadata fs.Metadata) (newDst fs.Directory, err error) { + do := f.Features().MkdirMetadata + if do == nil { + return nil, Mkdir(ctx, f, dir) + } + logName := fs.LogDirName(f, dir) + if SkipDestructive(ctx, logName, "make directory") { + return nil, nil + } + fs.Debugf(fs.LogDirName(f, dir), "Making directory with metadata") + newDst, err = do(ctx, dir, metadata) + if err != nil { + err = fs.CountError(ctx, err) + return nil, err + } + if mtime, ok := metadata["mtime"]; ok { + fs.Infof(logName, "Made directory with metadata (mtime=%s)", mtime) + } else { + fs.Infof(logName, "Made directory with metadata") + } + return newDst, err +} + +// MkdirModTime makes a destination directory or container with modtime +// +// It will try to make the directory with MkdirMetadata and if that +// succeeds it will return a non-nil newDst. In all other cases newDst +// will be nil. +// +// If the directory was created with MkDir then it will attempt to use +// Fs.DirSetModTime to update the directory modtime if available. +func MkdirModTime(ctx context.Context, f fs.Fs, dir string, modTime time.Time) (newDst fs.Directory, err error) { + logName := fs.LogDirName(f, dir) + if SkipDestructive(ctx, logName, "make directory") { + return nil, nil + } + metadata := fs.Metadata{ + "mtime": modTime.Format(time.RFC3339Nano), + } + newDst, err = MkdirMetadata(ctx, f, dir, metadata) + if err != nil { + return nil, err + } + if newDst != nil { + // The directory was created and we have logged already + return newDst, nil + } + // The directory was created with Mkdir then we should try to set the time + if do := f.Features().DirSetModTime; do != nil { + err = do(ctx, dir, modTime) + } + fs.Infof(logName, "Made directory with modification time %v", modTime) + return newDst, err +} + +// TryRmdir removes a container but not if not empty. It doesn't +// count errors but may return one. +func TryRmdir(ctx context.Context, f fs.Fs, dir string) error { + accounting.Stats(ctx).DeletedDirs(1) + if SkipDestructive(ctx, fs.LogDirName(f, dir), "remove directory") { + return nil + } + fs.Infof(fs.LogDirName(f, dir), "Removing directory") + return f.Rmdir(ctx, dir) +} + +// Rmdir removes a container but not if not empty +func Rmdir(ctx context.Context, f fs.Fs, dir string) error { + err := TryRmdir(ctx, f, dir) + if err != nil { + err = fs.CountError(ctx, err) + return err + } + return err +} + +// Purge removes a directory and all of its contents +func Purge(ctx context.Context, f fs.Fs, dir string) (err error) { + doFallbackPurge := true + if doPurge := f.Features().Purge; doPurge != nil { + doFallbackPurge = false + accounting.Stats(ctx).DeletedDirs(1) + if SkipDestructive(ctx, fs.LogDirName(f, dir), "purge directory") { + return nil + } + err = doPurge(ctx, dir) + if errors.Is(err, fs.ErrorCantPurge) { + doFallbackPurge = true + } + } + if doFallbackPurge { + // DeleteFiles and Rmdir observe --dry-run + err = DeleteFiles(ctx, listToChan(ctx, f, dir)) + if err != nil { + return err + } + err = Rmdirs(ctx, f, dir, false) + } + if err != nil { + err = fs.CountError(ctx, err) + return err + } + return nil +} + +// Delete removes all the contents of a container. Unlike Purge, it +// obeys includes and excludes. +func Delete(ctx context.Context, f fs.Fs) error { + ci := fs.GetConfig(ctx) + delChan := make(fs.ObjectsChan, ci.Checkers) + delErr := make(chan error, 1) + go func() { + delErr <- DeleteFiles(ctx, delChan) + }() + err := ListFn(ctx, f, func(o fs.Object) { + delChan <- o + }) + close(delChan) + delError := <-delErr + if err == nil { + err = delError + } + return err +} + +// RemoveExisting removes an existing file in a safe way so that it +// can be restored if the operation fails. +// +// This first detects if there is an existing file and renames it to a +// temporary name if there is. +// +// The returned cleanup function should be called on a defer statement +// with a pointer to the error returned. It will revert the changes if +// there is an error or delete the existing file if not. +func RemoveExisting(ctx context.Context, f fs.Fs, remote string, operation string) (cleanup func(*error), err error) { + existingObj, err := f.NewObject(ctx, remote) + if err != nil { + return func(*error) {}, nil + } + doMove := f.Features().Move + if doMove == nil { + return nil, fmt.Errorf("%s: destination file exists already and can't rename", operation) + } + + // Avoid making the leaf name longer if it's already lengthy to avoid + // trouble with file name length limits. + suffix := "." + random.String(8) + var remoteSaved string + if len(path.Base(remote)) > 100 { + remoteSaved = TruncateString(remote, len(remote)-len(suffix)) + suffix + } else { + remoteSaved = remote + suffix + } + + fs.Debugf(existingObj, "%s: renaming existing object to %q before starting", operation, remoteSaved) + existingObj, err = doMove(ctx, existingObj, remoteSaved) + if err != nil { + return nil, fmt.Errorf("%s: failed to rename existing file: %w", operation, err) + } + return func(perr *error) { + if *perr == nil { + fs.Debugf(existingObj, "%s: removing renamed existing file after operation", operation) + err := existingObj.Remove(ctx) + if err != nil { + *perr = fmt.Errorf("%s: failed to remove renamed existing file: %w", operation, err) + } + } else { + fs.Debugf(existingObj, "%s: renaming existing back after failed operation", operation) + _, renameErr := doMove(ctx, existingObj, remote) + if renameErr != nil { + fs.Errorf(existingObj, "%s: failed to restore existing file after failed operation: %v", operation, renameErr) + } + } + }, nil +} + +// listToChan will transfer all objects in the listing to the output +// +// If an error occurs, the error will be logged, and it will close the +// channel. +// +// If the error was ErrorDirNotFound then it will be ignored +func listToChan(ctx context.Context, f fs.Fs, dir string) fs.ObjectsChan { + ci := fs.GetConfig(ctx) + o := make(fs.ObjectsChan, ci.Checkers) + go func() { + defer close(o) + err := walk.ListR(ctx, f, dir, true, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error { + entries.ForObject(func(obj fs.Object) { + o <- obj + }) + return nil + }) + if err != nil && err != fs.ErrorDirNotFound { + err = fmt.Errorf("failed to list: %w", err) + err = fs.CountError(ctx, err) + fs.Errorf(nil, "%v", err) + } + }() + return o +} + +// CleanUp removes the trash for the Fs +func CleanUp(ctx context.Context, f fs.Fs) error { + doCleanUp := f.Features().CleanUp + if doCleanUp == nil { + return fmt.Errorf("%v doesn't support cleanup", f) + } + if SkipDestructive(ctx, f, "clean up old files") { + return nil + } + return doCleanUp(ctx) +} + +// wrap a Reader and a Closer together into a ReadCloser +type readCloser struct { + io.Reader + io.Closer +} + +// Cat any files to the io.Writer +// +// if offset == 0 it will be ignored +// if offset > 0 then the file will be seeked to that offset +// if offset < 0 then the file will be seeked that far from the end +// +// if count < 0 then it will be ignored +// if count >= 0 then only that many characters will be output +func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []byte) error { + var mu sync.Mutex + ci := fs.GetConfig(ctx) + return ListFn(ctx, f, func(o fs.Object) { + var err error + tr := accounting.Stats(ctx).NewTransfer(o, nil) + defer func() { + tr.Done(ctx, err) + }() + opt := fs.RangeOption{Start: offset, End: -1} + size := o.Size() + if opt.Start < 0 { + opt.Start += size + } + if count >= 0 { + opt.End = opt.Start + count - 1 + } + var options []fs.OpenOption + if opt.Start > 0 || opt.End >= 0 { + options = append(options, &opt) + } + for _, option := range ci.DownloadHeaders { + options = append(options, option) + } + var in io.ReadCloser + in, err = Open(ctx, o, options...) + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(o, "Failed to open: %v", err) + return + } + if count >= 0 { + in = &readCloser{Reader: &io.LimitedReader{R: in, N: count}, Closer: in} + } + in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer + // take the lock just before we output stuff, so at the last possible moment + mu.Lock() + defer mu.Unlock() + _, err = io.Copy(w, in) + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(o, "Failed to send to output: %v", err) + } + if len(sep) > 0 { + _, err = w.Write(sep) + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(o, "Failed to send separator to output: %v", err) + } + } + }) +} + +// Rcat reads data from the Reader until EOF and uploads it to a file on remote +// +// in is closed at the end of the transfer +func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time, meta fs.Metadata) (dst fs.Object, err error) { + return rcatSrc(ctx, fdst, dstFileName, in, modTime, meta, nil) +} + +// rcatSrc reads data from the Reader until EOF and uploads it to a file on remote +// +// in is closed at the end of the transfer +// +// Pass in fsrc if known or nil if not +func rcatSrc(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time, meta fs.Metadata, fsrc fs.Fs) (dst fs.Object, err error) { + if SkipDestructive(ctx, dstFileName, "upload from pipe") { + // prevents "broken pipe" errors + _, err = io.Copy(io.Discard, in) + return nil, err + } + + ci := fs.GetConfig(ctx) + tr := accounting.Stats(ctx).NewTransferRemoteSize(dstFileName, -1, nil, fdst) + defer func() { + tr.Done(ctx, err) + }() + var streamIn io.Reader = tr.Account(ctx, in).WithBuffer() + + readCounter := readers.NewCountingReader(streamIn) + var trackingIn io.Reader + var hasher *hash.MultiHasher + var options []fs.OpenOption + if !ci.IgnoreChecksum { + hashes := hash.NewHashSet(fdst.Hashes().GetOne()) // just pick one hash + hashOption := &fs.HashesOption{Hashes: hashes} + options = append(options, hashOption) + hasher, err = hash.NewMultiHasherTypes(hashes) + if err != nil { + return nil, err + } + trackingIn = io.TeeReader(readCounter, hasher) + } else { + trackingIn = readCounter + } + for _, option := range ci.UploadHeaders { + options = append(options, option) + } + if ci.MetadataSet != nil { + options = append(options, fs.MetadataOption(ci.MetadataSet)) + } + + // get the sums from the hasher if in use, or nil + getSums := func() (sums map[hash.Type]string) { + if hasher != nil { + sums = hasher.Sums() + } + return sums + } + + // Read the start of the input and check if it is small enough for direct upload + buf := make([]byte, ci.StreamingUploadCutoff) + fileIsSmall := false + if n, err := io.ReadFull(trackingIn, buf); err == io.EOF || err == io.ErrUnexpectedEOF { + fileIsSmall = true + buf = buf[:n] + } + + // Read the data we have already read in buf and any further unread + streamIn = io.MultiReader(bytes.NewReader(buf), trackingIn) + + doPutStream := fdst.Features().PutStream + + // Upload the input + if fileIsSmall || doPutStream == nil { + var rs io.ReadSeeker + if fileIsSmall { + fs.Debugf(fdst, "File to upload is small (%d bytes), uploading instead of streaming", len(buf)) + rs = bytes.NewReader(buf) + } else { + fs.Debugf(fdst, "Target remote doesn't support streaming uploads, creating temporary local FS to spool file") + spool, err := os.CreateTemp("", "rclone-spool") + if err != nil { + return nil, fmt.Errorf("failed to create temporary spool file: %v", err) + } + fileName := spool.Name() + defer func() { + err := spool.Close() + if err != nil { + fs.Errorf(fileName, "Failed to close temporary spool file: %v", err) + } + err = os.Remove(fileName) + if err != nil { + fs.Errorf(fileName, "Failed to delete temporary spool file: %v", err) + } + }() + _, err = io.Copy(spool, streamIn) + if err != nil { + return nil, fmt.Errorf("failed to copy to temporary spool file: %v", err) + } + rs = spool + } + // Upload with Put with retries - since we have downloaded the file we know the size, and the hashes + sums := getSums() + size := int64(readCounter.BytesRead()) + objInfo := object.NewStaticObjectInfo(dstFileName, modTime, size, false, sums, fsrc).WithMetadata(meta) + err = Retry(ctx, objInfo, ci.LowLevelRetries, func() error { + _, err = rs.Seek(0, io.SeekStart) + if err != nil { + return fmt.Errorf("failed to rewind temporary spool file: %v", err) + } + dst, err = fdst.Put(ctx, rs, objInfo, options...) + return err + }) + } else { + // Upload with PutStream with no retries + objInfo := object.NewStaticObjectInfo(dstFileName, modTime, -1, false, nil, fsrc).WithMetadata(meta) + dst, err = doPutStream(ctx, streamIn, objInfo, options...) + } + if err != nil { + return dst, err + } + + // Check transfer + sums := getSums() + opt := defaultEqualOpt(ctx) + if sums != nil { + // force --checksum on if we have hashes + opt.checkSum = true + } + src := object.NewStaticObjectInfo(dstFileName, modTime, int64(readCounter.BytesRead()), false, sums, fdst).WithMetadata(meta) + if !equal(ctx, src, dst, opt) { + err = fmt.Errorf("corrupted on transfer") + err = fs.CountError(ctx, err) + fs.Errorf(dst, "%v", err) + return dst, err + } + return dst, nil +} + +// PublicLink adds a "readable by anyone with link" permission on the given file or folder. +func PublicLink(ctx context.Context, f fs.Fs, remote string, expire fs.Duration, unlink bool) (string, error) { + doPublicLink := f.Features().PublicLink + if doPublicLink == nil { + return "", fmt.Errorf("%v doesn't support public links", f) + } + return doPublicLink(ctx, remote, expire, unlink) +} + +// Rmdirs removes any empty directories (or directories only +// containing empty directories) under f, including f. +// +// Rmdirs obeys the filters +func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error { + ci := fs.GetConfig(ctx) + fi := filter.GetConfig(ctx) + dirEmpty := make(map[string]bool) + dirEmpty[dir] = !leaveRoot + err := walk.Walk(ctx, f, dir, false, ci.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error { + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(f, "Failed to list %q: %v", dirPath, err) + return nil + } + for _, entry := range entries { + switch x := entry.(type) { + case fs.Directory: + // add a new directory as empty + dir := x.Remote() + _, found := dirEmpty[dir] + if !found { + dirEmpty[dir] = true + } + case fs.Object: + // mark the parents of the file as being non-empty + dir := x.Remote() + for dir != "" { + dir = path.Dir(dir) + if dir == "." || dir == "/" { + dir = "" + } + empty, found := dirEmpty[dir] + // End if we reach a directory which is non-empty + if found && !empty { + break + } + dirEmpty[dir] = false + } + } + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to rmdirs: %w", err) + } + + // Group directories to delete by level + var toDelete [][]string + for dir, empty := range dirEmpty { + if empty { + // If a filter matches the directory then that + // directory is a candidate for deletion + if fi.IncludeRemote(dir + "/") { + level := strings.Count(dir, "/") + 1 + // The root directory "" is at the top level + if dir == "" { + level = 0 + } + if len(toDelete) < level+1 { + toDelete = append(toDelete, make([][]string, level+1-len(toDelete))...) + } + toDelete[level] = append(toDelete[level], dir) + } + } + } + + errCount := errcount.New() + // Delete all directories at the same level in parallel + for level := len(toDelete) - 1; level >= 0; level-- { + dirs := toDelete[level] + if len(dirs) == 0 { + continue + } + fs.Debugf(nil, "removing %d level %d directories", len(dirs), level) + sort.Strings(dirs) + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(ci.Checkers) + for _, dir := range dirs { + // End early if error + if gCtx.Err() != nil { + break + } + dir := dir + g.Go(func() error { + err := TryRmdir(gCtx, f, dir) + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(dir, "Failed to rmdir: %v", err) + errCount.Add(err) + } + return nil // don't return errors, just count them + }) + } + err := g.Wait() + if err != nil { + return err + } + } + return errCount.Err("failed to remove directories") +} + +// GetCompareDest sets up --compare-dest +func GetCompareDest(ctx context.Context) (CompareDest []fs.Fs, err error) { + ci := fs.GetConfig(ctx) + CompareDest, err = cache.GetArr(ctx, ci.CompareDest) + if err != nil { + return nil, fserrors.FatalError(fmt.Errorf("failed to make fs for --compare-dest %q: %w", ci.CompareDest, err)) + } + return CompareDest, nil +} + +// compareDest checks --compare-dest to see if src needs to +// be copied +// +// Returns True if src is in --compare-dest +func compareDest(ctx context.Context, dst, src fs.Object, CompareDest fs.Fs) (NoNeedTransfer bool, err error) { + var remote string + if dst == nil { + remote = src.Remote() + } else { + remote = dst.Remote() + } + CompareDestFile, err := CompareDest.NewObject(ctx, remote) + switch err { + case fs.ErrorObjectNotFound: + return false, nil + case nil: + break + default: + return false, err + } + opt := defaultEqualOpt(ctx) + opt.updateModTime = false + if equal(ctx, src, CompareDestFile, opt) { + fs.Debugf(src, "Destination found in --compare-dest, skipping") + return true, nil + } + return false, nil +} + +// GetCopyDest sets up --copy-dest +func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest []fs.Fs, err error) { + ci := fs.GetConfig(ctx) + CopyDest, err = cache.GetArr(ctx, ci.CopyDest) + if err != nil { + return nil, fserrors.FatalError(fmt.Errorf("failed to make fs for --copy-dest %q: %w", ci.CopyDest, err)) + } + if !SameConfigArr(fdst, CopyDest) { + return nil, fserrors.FatalError(errors.New("parameter to --copy-dest has to be on the same remote as destination")) + } + for _, cf := range CopyDest { + if cf.Features().Copy == nil { + return nil, fserrors.FatalError(errors.New("can't use --copy-dest on a remote which doesn't support server side copy")) + } + } + + return CopyDest, nil +} + +// copyDest checks --copy-dest to see if src needs to +// be copied +// +// Returns True if src was copied from --copy-dest +func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) { + var remote string + if dst == nil { + remote = src.Remote() + } else { + remote = dst.Remote() + } + CopyDestFile, err := CopyDest.NewObject(ctx, remote) + switch err { + case fs.ErrorObjectNotFound: + return false, nil + case nil: + break + default: + return false, err + } + opt := defaultEqualOpt(ctx) + opt.updateModTime = false + if equal(ctx, src, CopyDestFile, opt) { + if dst == nil || !Equal(ctx, src, dst) { + if dst != nil && backupDir != nil { + err = MoveBackupDir(ctx, backupDir, dst) + if err != nil { + return false, fmt.Errorf("moving to --backup-dir failed: %w", err) + } + // If successful zero out the dstObj as it is no longer there + dst = nil + } + _, err := Copy(ctx, fdst, dst, remote, CopyDestFile) + if err != nil { + fs.Errorf(src, "Destination found in --copy-dest, error copying") + return false, nil + } + fs.Debugf(src, "Destination found in --copy-dest, using server-side copy") + return true, nil + } + fs.Debugf(src, "Unchanged skipping") + return true, nil + } + fs.Debugf(src, "Destination not found in --copy-dest") + return false, nil +} + +// CompareOrCopyDest checks --compare-dest and --copy-dest to see if src +// does not need to be copied +// +// Returns True if src does not need to be copied +func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest []fs.Fs, backupDir fs.Fs) (NoNeedTransfer bool, err error) { + ci := fs.GetConfig(ctx) + if len(ci.CompareDest) > 0 { + for _, compareF := range CompareOrCopyDest { + NoNeedTransfer, err := compareDest(ctx, dst, src, compareF) + if NoNeedTransfer || err != nil { + return NoNeedTransfer, err + } + } + } else if len(ci.CopyDest) > 0 { + for _, copyF := range CompareOrCopyDest { + NoNeedTransfer, err := copyDest(ctx, fdst, dst, src, copyF, backupDir) + if NoNeedTransfer || err != nil { + return NoNeedTransfer, err + } + } + } + return false, nil +} + +// NeedTransfer checks to see if src needs to be copied to dst using +// the current config. +// +// Returns a flag which indicates whether the file needs to be +// transferred or not. +func NeedTransfer(ctx context.Context, dst, src fs.Object) bool { + ci := fs.GetConfig(ctx) + logger, _ := GetLogger(ctx) + if dst == nil { + fs.Debugf(src, "Need to transfer - File not found at Destination") + logger(ctx, MissingOnDst, src, nil, nil) + return true + } + // If we should ignore existing files, don't transfer + if ci.IgnoreExisting { + fs.Debugf(src, "Destination exists, skipping") + logger(ctx, Match, src, dst, nil) + return false + } + // If we should upload unconditionally + if ci.IgnoreTimes { + fs.Debugf(src, "Transferring unconditionally as --ignore-times is in use") + logger(ctx, Differ, src, dst, nil) + return true + } + // If UpdateOlder is in effect, skip if dst is newer than src + if ci.UpdateOlder { + srcModTime := src.ModTime(ctx) + dstModTime := dst.ModTime(ctx) + dt := dstModTime.Sub(srcModTime) + // If have a mutually agreed precision then use that + modifyWindow := fs.GetModifyWindow(ctx, dst.Fs(), src.Fs()) + if modifyWindow == fs.ModTimeNotSupported { + // Otherwise use 1 second as a safe default as + // the resolution of the time a file was + // uploaded. + modifyWindow = time.Second + } + switch { + case dt >= modifyWindow: + fs.Debugf(src, "Destination is newer than source, skipping") + logger(ctx, Match, src, dst, nil) + return false + case dt <= -modifyWindow: + // force --checksum on for the check and do update modtimes by default + opt := defaultEqualOpt(ctx) + opt.forceModTimeMatch = true + if equal(ctx, src, dst, opt) { + fs.Debugf(src, "Unchanged skipping") + return false + } + default: + // Do a size only compare unless --checksum is set + opt := defaultEqualOpt(ctx) + opt.sizeOnly = !ci.CheckSum + if equal(ctx, src, dst, opt) { + fs.Debugf(src, "Destination mod time is within %v of source and files identical, skipping", modifyWindow) + return false + } + fs.Debugf(src, "Destination mod time is within %v of source but files differ, transferring", modifyWindow) + } + } else { + // Check to see if changed or not + equalFn, ok := ctx.Value(equalFnKey).(EqualFn) + if ok { + return !equalFn(ctx, src, dst) + } + if Equal(ctx, src, dst) && !SameObject(src, dst) { + fs.Debugf(src, "Unchanged skipping") + return false + } + } + return true +} + +// RcatSize reads data from the Reader until EOF and uploads it to a file on remote. +// Pass in size >=0 if known, <0 if not known +func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modTime time.Time, meta fs.Metadata) (dst fs.Object, err error) { + var obj fs.Object + + if size >= 0 { + var err error + // Size known use Put + tr := accounting.Stats(ctx).NewTransferRemoteSize(dstFileName, size, nil, fdst) + defer func() { + tr.Done(ctx, err) + }() + body := io.NopCloser(in) // we let the server close the body + in := tr.Account(ctx, body) // account the transfer (no buffering) + + if SkipDestructive(ctx, dstFileName, "upload from pipe") { + // prevents "broken pipe" errors + _, err = io.Copy(io.Discard, in) + return nil, err + } + + info := object.NewStaticObjectInfo(dstFileName, modTime, size, true, nil, fdst).WithMetadata(meta) + obj, err = fdst.Put(ctx, in, info) + if err != nil { + fs.Errorf(dstFileName, "Post request put error: %v", err) + + return nil, err + } + } else { + // Size unknown use Rcat + obj, err = Rcat(ctx, fdst, dstFileName, in, modTime, meta) + if err != nil { + fs.Errorf(dstFileName, "Post request rcat error: %v", err) + + return nil, err + } + } + + return obj, nil +} + +// copyURLFunc is called from CopyURLFn +type copyURLFunc func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) + +// copyURLFn copies the data from the url to the function supplied +func copyURLFn(ctx context.Context, dstFileName string, url string, autoFilename, dstFileNameFromHeader bool, fn copyURLFunc) (err error) { + client := fshttp.NewClient(ctx) + resp, err := client.Get(url) + if err != nil { + return err + } + defer fs.CheckClose(resp.Body, &err) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("CopyURL failed: %s", resp.Status) + } + modTime, err := http.ParseTime(resp.Header.Get("Last-Modified")) + if err != nil { + modTime = time.Now() + } + if autoFilename { + if dstFileNameFromHeader { + _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) + headerFilename := path.Base(strings.ReplaceAll(params["filename"], "\\", "/")) + if err != nil || headerFilename == "" { + return fmt.Errorf("CopyURL failed: filename not found in the Content-Disposition header") + } + fs.Debugf(headerFilename, "filename found in Content-Disposition header.") + return fn(ctx, headerFilename, resp.Body, resp.ContentLength, modTime) + } + + dstFileName = path.Base(resp.Request.URL.Path) + if dstFileName == "." || dstFileName == "/" { + return fmt.Errorf("CopyURL failed: file name wasn't found in url") + } + fs.Debugf(dstFileName, "File name found in url") + } + return fn(ctx, dstFileName, resp.Body, resp.ContentLength, modTime) +} + +// CopyURL copies the data from the url to (fdst, dstFileName) +func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, autoFilename, dstFileNameFromHeader bool, noClobber bool) (dst fs.Object, err error) { + err = copyURLFn(ctx, dstFileName, url, autoFilename, dstFileNameFromHeader, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) { + if noClobber { + _, err = fdst.NewObject(ctx, dstFileName) + if err == nil { + return errors.New("CopyURL failed: file already exist") + } + } + dst, err = RcatSize(ctx, fdst, dstFileName, in, size, modTime, nil) + return err + }) + return dst, err +} + +// CopyURLToWriter copies the data from the url to the io.Writer supplied +func CopyURLToWriter(ctx context.Context, url string, out io.Writer) (err error) { + return copyURLFn(ctx, "", url, false, false, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) { + _, err = io.Copy(out, in) + return err + }) +} + +// BackupDir returns the correctly configured --backup-dir +func BackupDir(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, srcFileName string) (backupDir fs.Fs, err error) { + ci := fs.GetConfig(ctx) + if ci.BackupDir != "" { + backupDir, err = cache.Get(ctx, ci.BackupDir) + if err != nil { + return nil, fserrors.FatalError(fmt.Errorf("failed to make fs for --backup-dir %q: %w", ci.BackupDir, err)) + } + if !SameConfig(fdst, backupDir) { + return nil, fserrors.FatalError(errors.New("parameter to --backup-dir has to be on the same remote as destination")) + } + if srcFileName == "" { + if OverlappingFilterCheck(ctx, backupDir, fdst) { + return nil, fserrors.FatalError(errors.New("destination and parameter to --backup-dir mustn't overlap")) + } + if OverlappingFilterCheck(ctx, backupDir, fsrc) { + return nil, fserrors.FatalError(errors.New("source and parameter to --backup-dir mustn't overlap")) + } + } else if ci.Suffix == "" { + if SameDir(fdst, backupDir) { + return nil, fserrors.FatalError(errors.New("destination and parameter to --backup-dir mustn't be the same")) + } + if SameDir(fsrc, backupDir) { + return nil, fserrors.FatalError(errors.New("source and parameter to --backup-dir mustn't be the same")) + } + } + } else if ci.Suffix != "" { + // --backup-dir is not set but --suffix is - use the destination as the backupDir + backupDir = fdst + } else { + return nil, fserrors.FatalError(errors.New("internal error: BackupDir called when --backup-dir and --suffix both empty")) + } + if !CanServerSideMove(backupDir) { + return nil, fserrors.FatalError(errors.New("can't use --backup-dir on a remote which doesn't support server-side move or copy")) + } + return backupDir, nil +} + +// MoveBackupDir moves a file to the backup dir +func MoveBackupDir(ctx context.Context, backupDir fs.Fs, dst fs.Object) (err error) { + remoteWithSuffix := SuffixName(ctx, dst.Remote()) + overwritten, _ := backupDir.NewObject(ctx, remoteWithSuffix) + _, err = Move(ctx, backupDir, overwritten, remoteWithSuffix, dst) + return err +} + +// needsMoveCaseInsensitive returns true if moveCaseInsensitive is needed +func needsMoveCaseInsensitive(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) bool { + dstFilePath := path.Join(fdst.Root(), dstFileName) + srcFilePath := path.Join(fsrc.Root(), srcFileName) + if !cp && fdst.Name() == fsrc.Name() && dstFileName != srcFileName && norm.NFC.String(dstFilePath) == norm.NFC.String(srcFilePath) { + return true + } + return !cp && fdst.Name() == fsrc.Name() && fdst.Features().CaseInsensitive && dstFileName != srcFileName && strings.EqualFold(dstFilePath, srcFilePath) +} + +// MoveCaseInsensitive handles changing case of a file on a case insensitive remote. +// This will move the file to a temporary name then +// move it back to the intended destination. This is required +// to avoid issues with certain remotes and avoid file deletion. +// returns nil, nil if !needsMoveCaseInsensitive. +// this does not account a transfer -- the caller should do that if desired. +func MoveCaseInsensitive(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool, srcObj fs.Object) (newDst fs.Object, err error) { + logger, _ := GetLogger(ctx) + + // Choose operations + Op := MoveTransfer + if cp { + Op = Copy + } + + if SkipDestructive(ctx, srcFileName, "rename to "+dstFileName) { + // avoid fatalpanic on --dry-run (trying to access non-existent tmpObj) + return nil, nil + } + // Create random name to temporarily move file to + tmpObjName := dstFileName + "-rclone-move-" + random.String(8) + tmpObjFail, err := fdst.NewObject(ctx, tmpObjName) + if err != fs.ErrorObjectNotFound { + if err == nil { + logger(ctx, TransferError, nil, tmpObjFail, err) + return nil, errors.New("found an already existing file with a randomly generated name. Try the operation again") + } + logger(ctx, TransferError, nil, tmpObjFail, err) + return nil, fmt.Errorf("error while attempting to move file to a temporary location: %w", err) + } + fs.Debugf(srcObj, "moving to %v", tmpObjName) + tmpObj, err := Op(ctx, fdst, nil, tmpObjName, srcObj) + if err != nil { + logger(ctx, TransferError, srcObj, tmpObj, err) + return nil, fmt.Errorf("error while moving file to temporary location: %w", err) + } + fs.Debugf(srcObj, "moving to %v", dstFileName) + newDst, err = Op(ctx, fdst, nil, dstFileName, tmpObj) + logger(ctx, MissingOnDst, tmpObj, nil, err) + return newDst, err +} + +// moveOrCopyFile moves or copies a single file possibly to a new name +func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool, allowOverlap bool) (err error) { + ci := fs.GetConfig(ctx) + logger, usingLogger := GetLogger(ctx) + dstFilePath := path.Join(fdst.Root(), dstFileName) + srcFilePath := path.Join(fsrc.Root(), srcFileName) + if fdst.Name() == fsrc.Name() && dstFilePath == srcFilePath && !allowOverlap { + fs.Debugf(fdst, "don't need to copy/move %s, it is already at target location", dstFileName) + if usingLogger { + srcObj, _ := fsrc.NewObject(ctx, srcFileName) + dstObj, _ := fsrc.NewObject(ctx, dstFileName) + logger(ctx, Match, srcObj, dstObj, nil) + } + return nil + } + + // Choose operations + Op := MoveTransfer + if cp { + Op = Copy + } + + // Find src object + srcObj, err := fsrc.NewObject(ctx, srcFileName) + if err != nil { + logger(ctx, TransferError, srcObj, nil, err) + return err + } + + // Find dst object if it exists + var dstObj fs.Object + if !ci.NoCheckDest { + dstObj, err = fdst.NewObject(ctx, dstFileName) + if errors.Is(err, fs.ErrorObjectNotFound) { + dstObj = nil + } else if err != nil { + logger(ctx, TransferError, nil, dstObj, err) + return err + } + } + + // Special case for changing case of a file on a case insensitive remote + // This will move the file to a temporary name then + // move it back to the intended destination. This is required + // to avoid issues with certain remotes and avoid file deletion. + if needsMoveCaseInsensitive(fdst, fsrc, dstFileName, srcFileName, cp) { + tr := accounting.Stats(ctx).NewTransfer(srcObj, fdst) + defer func() { + tr.Done(ctx, err) + }() + _, err = MoveCaseInsensitive(ctx, fdst, fsrc, dstFileName, srcFileName, cp, srcObj) + return err + } + + var backupDir fs.Fs + var copyDestDir []fs.Fs + if ci.BackupDir != "" || ci.Suffix != "" { + backupDir, err = BackupDir(ctx, fdst, fsrc, srcFileName) + if err != nil { + return fmt.Errorf("creating Fs for --backup-dir failed: %w", err) + } + } + if len(ci.CompareDest) > 0 { + copyDestDir, err = GetCompareDest(ctx) + if err != nil { + return err + } + } else if len(ci.CopyDest) > 0 { + copyDestDir, err = GetCopyDest(ctx, fdst) + if err != nil { + return err + } + } + needTransfer := NeedTransfer(ctx, dstObj, srcObj) + if needTransfer { + NoNeedTransfer, err := CompareOrCopyDest(ctx, fdst, dstObj, srcObj, copyDestDir, backupDir) + if err != nil { + return err + } + if NoNeedTransfer { + needTransfer = false + } + } + if needTransfer { + // If destination already exists, then we must move it into --backup-dir if required + if dstObj != nil && backupDir != nil { + err = MoveBackupDir(ctx, backupDir, dstObj) + if err != nil { + logger(ctx, TransferError, dstObj, nil, err) + return fmt.Errorf("moving to --backup-dir failed: %w", err) + } + // If successful zero out the dstObj as it is no longer there + logger(ctx, MissingOnDst, dstObj, nil, nil) + dstObj = nil + } + + _, err = Op(ctx, fdst, dstObj, dstFileName, srcObj) + } else if !cp { + if ci.IgnoreExisting { + fs.Debugf(srcObj, "Not removing source file as destination file exists and --ignore-existing is set") + logger(ctx, Match, srcObj, dstObj, nil) + } else if !SameObject(srcObj, dstObj) { + err = DeleteFile(ctx, srcObj) + logger(ctx, Differ, srcObj, dstObj, nil) + } + } + return err +} + +// MoveFile moves a single file possibly to a new name +// +// This is treated as a transfer. +func MoveFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) { + return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, false, false) +} + +// TransformFile transforms a file in place using --name-transform +// +// This is treated as a transfer. +func TransformFile(ctx context.Context, fdst fs.Fs, srcFileName string) (err error) { + return moveOrCopyFile(ctx, fdst, fdst, srcFileName, srcFileName, false, true) +} + +// SetTier changes tier of object in remote +func SetTier(ctx context.Context, fsrc fs.Fs, tier string) error { + return ListFn(ctx, fsrc, func(o fs.Object) { + objImpl, ok := o.(fs.SetTierer) + if !ok { + fs.Errorf(fsrc, "Remote object does not implement SetTier") + return + } + err := objImpl.SetTier(tier) + if err != nil { + fs.Errorf(fsrc, "Failed to do SetTier, %v", err) + } + }) +} + +// SetTierFile changes tier of a single file in remote +func SetTierFile(ctx context.Context, o fs.Object, tier string) error { + do, ok := o.(fs.SetTierer) + if !ok { + return errors.New("remote object does not implement SetTier") + } + err := do.SetTier(tier) + if err != nil { + fs.Errorf(o, "Failed to do SetTier, %v", err) + return err + } + return nil +} + +// TouchDir touches every file in directory with time t +func TouchDir(ctx context.Context, f fs.Fs, remote string, t time.Time, recursive bool) error { + ci := fs.GetConfig(ctx) + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(ci.Transfers) + err := walk.ListR(ctx, f, remote, false, ConfigMaxDepth(ctx, recursive), walk.ListObjects, func(entries fs.DirEntries) error { + entries.ForObject(func(o fs.Object) { + if !SkipDestructive(ctx, o, "touch") { + g.Go(func() error { + fs.Debugf(f, "Touching %q", o.Remote()) + err := o.SetModTime(gCtx, t) + if err != nil { + err = fmt.Errorf("failed to touch: %w", err) + err = fs.CountError(gCtx, err) + fs.Errorf(o, "%v", err) + } + return nil + }) + } + }) + return nil + }) + _ = g.Wait() + return err +} + +// ListFormat defines files information print format +type ListFormat struct { + separator string + dirSlash bool + absolute bool + output []func(entry *ListJSONItem) string + csv *csv.Writer + buf bytes.Buffer +} + +// SetSeparator changes separator in struct +func (l *ListFormat) SetSeparator(separator string) { + l.separator = separator +} + +// SetDirSlash defines if slash should be printed +func (l *ListFormat) SetDirSlash(dirSlash bool) { + l.dirSlash = dirSlash +} + +// SetAbsolute prints a leading slash in front of path names +func (l *ListFormat) SetAbsolute(absolute bool) { + l.absolute = absolute +} + +// SetCSV defines if the output should be csv +// +// Note that you should call SetSeparator before this if you want a +// custom separator +func (l *ListFormat) SetCSV(useCSV bool) { + if useCSV { + l.csv = csv.NewWriter(&l.buf) + if l.separator != "" { + l.csv.Comma = []rune(l.separator)[0] + } + } else { + l.csv = nil + } +} + +// SetOutput sets functions used to create files information +func (l *ListFormat) SetOutput(output []func(entry *ListJSONItem) string) { + l.output = output +} + +// AddModTime adds file's Mod Time to output +func (l *ListFormat) AddModTime(timeFormat string) { + switch timeFormat { + case "": + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.ModTime.When.Local().Format("2006-01-02 15:04:05") + }) + case "unix": + l.AppendOutput(func(entry *ListJSONItem) string { + return fmt.Sprint(entry.ModTime.When.Unix()) + }) + case "unixnano": + l.AppendOutput(func(entry *ListJSONItem) string { + return fmt.Sprint(entry.ModTime.When.UnixNano()) + }) + default: + timeFormat = transform.TimeFormat(timeFormat) + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.ModTime.When.Local().Format(timeFormat) + }) + } +} + +// AddSize adds file's size to output +func (l *ListFormat) AddSize() { + l.AppendOutput(func(entry *ListJSONItem) string { + return strconv.FormatInt(entry.Size, 10) + }) +} + +// normalisePath makes sure the path has the correct slashes for the current mode +func (l *ListFormat) normalisePath(entry *ListJSONItem, remote string) string { + if l.absolute && !strings.HasPrefix(remote, "/") { + remote = "/" + remote + } + if entry.IsDir && l.dirSlash { + remote += "/" + } + return remote +} + +// AddPath adds path to file to output +func (l *ListFormat) AddPath() { + l.AppendOutput(func(entry *ListJSONItem) string { + return l.normalisePath(entry, entry.Path) + }) +} + +// AddEncrypted adds the encrypted path to file to output +func (l *ListFormat) AddEncrypted() { + l.AppendOutput(func(entry *ListJSONItem) string { + return l.normalisePath(entry, entry.Encrypted) + }) +} + +// AddHash adds the hash of the type given to the output +func (l *ListFormat) AddHash(ht hash.Type) { + hashName := ht.String() + l.AppendOutput(func(entry *ListJSONItem) string { + if entry.IsDir { + return "" + } + return entry.Hashes[hashName] + }) +} + +// AddID adds file's ID to the output if known +func (l *ListFormat) AddID() { + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.ID + }) +} + +// AddOrigID adds file's Original ID to the output if known +func (l *ListFormat) AddOrigID() { + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.OrigID + }) +} + +// AddTier adds file's Tier to the output if known +func (l *ListFormat) AddTier() { + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.Tier + }) +} + +// AddMimeType adds file's MimeType to the output if known +func (l *ListFormat) AddMimeType() { + l.AppendOutput(func(entry *ListJSONItem) string { + return entry.MimeType + }) +} + +// AddMetadata adds file's Metadata to the output if known +func (l *ListFormat) AddMetadata() { + l.AppendOutput(func(entry *ListJSONItem) string { + metadata := entry.Metadata + if metadata == nil { + metadata = make(fs.Metadata) + } + out, err := json.Marshal(metadata) + if err != nil { + return fmt.Sprintf("Failed to read metadata: %v", err.Error()) + } + return string(out) + }) +} + +// AppendOutput adds string generated by specific function to printed output +func (l *ListFormat) AppendOutput(functionToAppend func(item *ListJSONItem) string) { + l.output = append(l.output, functionToAppend) +} + +// Format prints information about the DirEntry in the format defined +func (l *ListFormat) Format(entry *ListJSONItem) (result string) { + var out []string + for _, fun := range l.output { + out = append(out, fun(entry)) + } + if l.csv != nil { + l.buf.Reset() + _ = l.csv.Write(out) // can't fail writing to bytes.Buffer + l.csv.Flush() + result = strings.TrimRight(l.buf.String(), "\n") + } else { + result = strings.Join(out, l.separator) + } + return result +} + +// FormatForLSFPrecision Returns a time format for the given precision +func FormatForLSFPrecision(precision time.Duration) string { + switch { + case precision <= time.Nanosecond: + return "2006-01-02 15:04:05.000000000" + case precision <= 10*time.Nanosecond: + return "2006-01-02 15:04:05.00000000" + case precision <= 100*time.Nanosecond: + return "2006-01-02 15:04:05.0000000" + case precision <= time.Microsecond: + return "2006-01-02 15:04:05.000000" + case precision <= 10*time.Microsecond: + return "2006-01-02 15:04:05.00000" + case precision <= 100*time.Microsecond: + return "2006-01-02 15:04:05.0000" + case precision <= time.Millisecond: + return "2006-01-02 15:04:05.000" + case precision <= 10*time.Millisecond: + return "2006-01-02 15:04:05.00" + case precision <= 100*time.Millisecond: + return "2006-01-02 15:04:05.0" + } + return "2006-01-02 15:04:05" +} + +// DirMove renames srcRemote to dstRemote +// +// It does this by loading the directory tree into memory (using ListR +// if available) and doing renames in parallel. +func DirMove(ctx context.Context, f fs.Fs, srcRemote, dstRemote string) (err error) { + ci := fs.GetConfig(ctx) + + if SkipDestructive(ctx, srcRemote, "dirMove") { + accounting.Stats(ctx).Renames(1) + return nil + } + + // Use DirMove if possible + if doDirMove := f.Features().DirMove; doDirMove != nil { + err = doDirMove(ctx, f, srcRemote, dstRemote) + if err == nil { + accounting.Stats(ctx).Renames(1) + } + if err != fs.ErrorCantDirMove && err != fs.ErrorDirExists { + return err + } + fs.Infof(f, "Can't DirMove - falling back to file moves: %v", err) + } + + // Load the directory tree into memory + tree, err := walk.NewDirTree(ctx, f, srcRemote, true, -1) + if err != nil { + return fmt.Errorf("RenameDir tree walk: %w", err) + } + + // Get the directories in sorted order + dirs := tree.Dirs() + + // Make the destination directories - must be done in order not in parallel + for _, dir := range dirs { + dstPath := dstRemote + dir[len(srcRemote):] + err := f.Mkdir(ctx, dstPath) + if err != nil { + return fmt.Errorf("RenameDir mkdir: %w", err) + } + } + + // Rename the files in parallel + type rename struct { + o fs.Object + newPath string + } + renames := make(chan rename, ci.Checkers) + g, gCtx := errgroup.WithContext(context.Background()) + for range ci.Checkers { + g.Go(func() error { + for job := range renames { + dstOverwritten, _ := f.NewObject(gCtx, job.newPath) + _, err := Move(gCtx, f, dstOverwritten, job.newPath, job.o) + if err != nil { + return err + } + select { + case <-gCtx.Done(): + return gCtx.Err() + default: + } + + } + return nil + }) + } + for dir, entries := range tree { + dstPath := dstRemote + dir[len(srcRemote):] + for _, entry := range entries { + if o, ok := entry.(fs.Object); ok { + renames <- rename{o, path.Join(dstPath, path.Base(o.Remote()))} + } + } + } + close(renames) + err = g.Wait() + if err != nil { + return fmt.Errorf("RenameDir renames: %w", err) + } + + // Remove the source directories in reverse order + for i := len(dirs) - 1; i >= 0; i-- { + err := f.Rmdir(ctx, dirs[i]) + if err != nil { + return fmt.Errorf("RenameDir rmdir: %w", err) + } + } + + return nil +} + +// DirMoveCaseInsensitive does DirMove in two steps (to temp name, then real name) +// which is necessary for some case-insensitive backends +func DirMoveCaseInsensitive(ctx context.Context, f fs.Fs, srcRemote, dstRemote string) (err error) { + tmpDstRemote := dstRemote + "-rclone-move-" + random.String(8) + err = DirMove(ctx, f, srcRemote, tmpDstRemote) + if err != nil { + return err + } + return DirMove(ctx, f, tmpDstRemote, dstRemote) +} + +// FsInfo provides information about a remote +type FsInfo struct { + // Name of the remote (as passed into NewFs) + Name string + + // Root of the remote (as passed into NewFs) + Root string + + // String returns a description of the FS + String string + + // Precision of the ModTimes in this Fs in Nanoseconds + Precision time.Duration + + // Returns the supported hash types of the filesystem + Hashes []string + + // Features returns the optional features of this Fs + Features map[string]bool + + // MetadataInfo returns info about the metadata for this backend + MetadataInfo *fs.MetadataInfo +} + +// GetFsInfo gets the information (FsInfo) about a given Fs +func GetFsInfo(f fs.Fs) *FsInfo { + features := f.Features() + info := &FsInfo{ + Name: f.Name(), + Root: f.Root(), + String: f.String(), + Precision: f.Precision(), + Hashes: make([]string, 0, 4), + Features: features.Enabled(), + MetadataInfo: nil, + } + for _, hashType := range f.Hashes().Array() { + info.Hashes = append(info.Hashes, hashType.String()) + } + fsInfo, _, _, _, err := fs.ParseRemote(fs.ConfigString(f)) + if err == nil && fsInfo != nil && fsInfo.MetadataInfo != nil { + info.MetadataInfo = fsInfo.MetadataInfo + } + return info +} + +var ( + interactiveMu sync.Mutex // protects the following variables + skipped = map[string]bool{} +) + +// skipDestructiveChoose asks the user which action to take +// +// Call with interactiveMu held +func skipDestructiveChoose(ctx context.Context, subject any, action string) (skip bool) { + // Lock the StdoutMutex - must not call fs.Log anything + // otherwise it will deadlock with --interactive --progress + StdoutMutex.Lock() + + fmt.Printf("\nrclone: %s \"%v\"?\n", action, subject) + i := config.CommandDefault([]string{ + "yYes, this is OK", + "nNo, skip this", + fmt.Sprintf("sSkip all %s operations with no more questions", action), + fmt.Sprintf("!Do all %s operations with no more questions", action), + "qExit rclone now.", + }, 0) + + StdoutMutex.Unlock() + + switch i { + case 'y': + skip = false + case 'n': + skip = true + case 's': + skip = true + skipped[action] = true + fs.Logf(nil, "Skipping all %s operations from now on without asking", action) + case '!': + skip = false + skipped[action] = false + fs.Logf(nil, "Doing all %s operations from now on without asking", action) + case 'q': + fs.Logf(nil, "Quitting rclone now") + atexit.Run() + os.Exit(0) + default: + skip = true + fs.Errorf(nil, "Bad choice %c", i) + } + return skip +} + +// SkipDestructive should be called whenever rclone is about to do an destructive operation. +// +// It will check the --dry-run flag and it will ask the user if the --interactive flag is set. +// +// subject should be the object or directory in use +// +// action should be a descriptive word or short phrase +// +// Together they should make sense in this sentence: "Rclone is about +// to action subject". +func SkipDestructive(ctx context.Context, subject any, action string) (skip bool) { + var flag string + ci := fs.GetConfig(ctx) + switch { + case ci.DryRun: + flag = "--dry-run" + skip = true + case ci.Interactive: + flag = "--interactive" + interactiveMu.Lock() + defer interactiveMu.Unlock() + var found bool + skip, found = skipped[action] + if !found { + skip = skipDestructiveChoose(ctx, subject, action) + } + default: + return false + } + if skip { + size := int64(-1) + if do, ok := subject.(interface{ Size() int64 }); ok { + size = do.Size() + } + if size >= 0 { + fs.Logf(subject, "Skipped %s as %s is set (size %v)", fs.LogValue("skipped", action), flag, fs.LogValue("size", fs.SizeSuffix(size))) + } else { + fs.Logf(subject, "Skipped %s as %s is set", fs.LogValue("skipped", action), flag) + } + } + return skip +} + +// Return the best way of describing the directory for the logs +func dirName(f fs.Fs, dst fs.Directory, dir string) any { + if dst != nil { + if dst.Remote() != "" { + return dst + } + // Root is described as the Fs + return f + } + if dir != "" { + return dir + } + // Root is described as the Fs + return f +} + +// CopyDirMetadata copies the src directory to dst or f if nil. If dst is nil then it uses +// dir as the name of the new directory. +// +// It returns the destination directory if possible. Note that this may +// be nil. +func CopyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory, err error) { + ci := fs.GetConfig(ctx) + logName := dirName(f, dst, dir) + if SkipDestructive(ctx, logName, "update directory metadata") { + return nil, nil + } + + // Options for the directory metadata + options := []fs.OpenOption{} + if ci.MetadataSet != nil { + options = append(options, fs.MetadataOption(ci.MetadataSet)) + } + + // Read metadata from src and add options and use metadata mapper + metadata, err := fs.GetMetadataOptions(ctx, f, src, options) + if err != nil { + return nil, err + } + + // Fall back to ModTime if metadata not available + if metadata == nil { + metadata = fs.Metadata{} + } + if metadata["mtime"] == "" { + metadata["mtime"] = src.ModTime(ctx).Format(time.RFC3339Nano) + } + + // Now set the metadata + if dst == nil { + do := f.Features().MkdirMetadata + if do == nil { + return nil, fmt.Errorf("internal error: expecting %v to have MkdirMetadata method: %w", f, fs.ErrorNotImplemented) + } + newDst, err = do(ctx, dir, metadata) + } else { + do, ok := dst.(fs.SetMetadataer) + if !ok { + return nil, fmt.Errorf("internal error: expecting directory %s (%T) from %v to have SetMetadata method: %w", logName, dst, f, fs.ErrorNotImplemented) + } + err = do.SetMetadata(ctx, metadata) + newDst = dst + } + if err != nil { + return nil, err + } + fs.Infof(logName, "Updated directory metadata") + return newDst, nil +} + +// SetDirModTime sets the modtime on dst or dir +// +// If dst is nil then it uses dir as the name of the directory. +// +// It returns the destination directory if possible. Note that this +// may be nil. +// +// It does not create the directory. +func SetDirModTime(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, modTime time.Time) (newDst fs.Directory, err error) { + logName := dirName(f, dst, dir) + ci := fs.GetConfig(ctx) + if ci.NoUpdateDirModTime { + fs.Debugf(logName, "Skipping set directory modification time as --no-update-dir-modtime is set") + return nil, nil + } + if SkipDestructive(ctx, logName, "set directory modification time") { + return nil, nil + } + if dst != nil { + dir = dst.Remote() + } + + // Try to set the ModTime with the Directory.SetModTime method first as this is the most efficient + if dst != nil { + if do, ok := dst.(fs.SetModTimer); ok { + err := do.SetModTime(ctx, modTime) + if errors.Is(err, fs.ErrorNotImplemented) { + // Fall through and run the code below if not implemented + // This can happen for fs.DirWrapper instances + } else if err != nil { + return dst, err + } else { + fs.Infof(logName, "Set directory modification time (using SetModTime)") + return dst, nil + } + } + } + + // Next try to set the ModTime with the Fs.DirSetModTime method as this works for non-metadata backends + if do := f.Features().DirSetModTime; do != nil { + err := do(ctx, dir, modTime) + if err != nil { + return dst, err + } + fs.Infof(logName, "Set directory modification time (using DirSetModTime)") + return dst, nil + } + + // Something should have worked so return an error + return nil, fmt.Errorf("no method to set directory modtime found for %v (%T): %w", f, dst, fs.ErrorNotImplemented) +} |