aboutsummaryrefslogtreecommitdiff
path: root/fs/operations/logger.go
diff options
context:
space:
mode:
Diffstat (limited to 'fs/operations/logger.go')
-rw-r--r--fs/operations/logger.go384
1 files changed, 384 insertions, 0 deletions
diff --git a/fs/operations/logger.go b/fs/operations/logger.go
new file mode 100644
index 0000000..0d48220
--- /dev/null
+++ b/fs/operations/logger.go
@@ -0,0 +1,384 @@
+package operations
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ mutex "sync"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/spf13/pflag"
+)
+
+// Sigil represents the rune (-+=*!?) used by Logger to categorize files by their match/differ/missing status.
+type Sigil rune
+
+// String converts sigil to more human-readable string
+func (sigil Sigil) String() string {
+ switch sigil {
+ case '-':
+ return "MissingOnSrc"
+ case '+':
+ return "MissingOnDst"
+ case '=':
+ return "Match"
+ case '*':
+ return "Differ"
+ case '!':
+ return "Error"
+ // case '.':
+ // return "Completed"
+ case '?':
+ return "Other"
+ }
+ return "unknown"
+}
+
+// Writer directs traffic from sigil -> LoggerOpt.Writer
+func (sigil Sigil) Writer(opt LoggerOpt) io.Writer {
+ switch sigil {
+ case '-':
+ return opt.MissingOnSrc
+ case '+':
+ return opt.MissingOnDst
+ case '=':
+ return opt.Match
+ case '*':
+ return opt.Differ
+ case '!':
+ return opt.Error
+ }
+ return nil
+}
+
+// Sigil constants
+const (
+ MissingOnSrc Sigil = '-'
+ MissingOnDst Sigil = '+'
+ Match Sigil = '='
+ Differ Sigil = '*'
+ TransferError Sigil = '!'
+ Other Sigil = '?' // reserved but not currently used
+)
+
+// LoggerFn uses fs.DirEntry instead of fs.Object so it can include Dirs
+// For LoggerFn example, see bisync.WriteResults() or sync.SyncLoggerFn()
+// Usage example: s.logger(ctx, operations.Differ, src, dst, nil)
+type LoggerFn func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error)
+type loggerContextKey struct{}
+type loggerOptContextKey struct{}
+
+var loggerKey = loggerContextKey{}
+var loggerOptKey = loggerOptContextKey{}
+
+// LoggerOpt contains options for the Sync Logger functions
+// TODO: refactor Check in here too?
+type LoggerOpt struct {
+ // Fdst, Fsrc fs.Fs // fses to check
+ // Check checkFn // function to use for checking
+ // OneWay bool // one way only?
+ LoggerFn LoggerFn // function to use for logging
+ Combined io.Writer // a file with file names with leading sigils
+ MissingOnSrc io.Writer // files only in the destination
+ MissingOnDst io.Writer // files only in the source
+ Match io.Writer // matching files
+ Differ io.Writer // differing files
+ Error io.Writer // files with errors of some kind
+ DestAfter io.Writer // files that exist on the destination post-sync
+ JSON *bytes.Buffer // used by bisync to read/write struct as JSON
+ DeleteModeOff bool //affects whether Logger expects MissingOnSrc to be deleted
+
+ // lsf options for destAfter
+ ListFormat ListFormat
+ JSONOpt ListJSONOpt
+ LJ *listJSON
+ Format string
+ TimeFormat string
+ Separator string
+ DirSlash bool
+ // Recurse bool
+ HashType hash.Type
+ FilesOnly bool
+ DirsOnly bool
+ Csv bool
+ Absolute bool
+}
+
+// NewDefaultLoggerFn creates a logger function that writes the sigil and path to configured files that match the sigil
+func NewDefaultLoggerFn(opt *LoggerOpt) LoggerFn {
+ var lock mutex.Mutex
+
+ return func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {
+ lock.Lock()
+ defer lock.Unlock()
+
+ if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil {
+ opt.PrintDestAfter(ctx, sigil, src, dst, err)
+ return
+ }
+
+ _, srcOk := src.(fs.Object)
+ _, dstOk := dst.(fs.Object)
+ var filename string
+ if !srcOk && !dstOk {
+ return
+ } else if srcOk && !dstOk {
+ filename = src.String()
+ } else {
+ filename = dst.String()
+ }
+
+ if sigil.Writer(*opt) != nil {
+ SyncFprintf(sigil.Writer(*opt), "%s\n", filename)
+ }
+ if opt.Combined != nil {
+ SyncFprintf(opt.Combined, "%c %s\n", sigil, filename)
+ fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename)
+ }
+ if opt.DestAfter != nil {
+ opt.PrintDestAfter(ctx, sigil, src, dst, err)
+ }
+ }
+}
+
+// WithLogger stores logger in ctx and returns a copy of ctx in which loggerKey = logger
+func WithLogger(ctx context.Context, logger LoggerFn) context.Context {
+ return context.WithValue(ctx, loggerKey, logger)
+}
+
+// WithLoggerOpt stores loggerOpt in ctx and returns a copy of ctx in which loggerOptKey = loggerOpt
+func WithLoggerOpt(ctx context.Context, loggerOpt LoggerOpt) context.Context {
+ return context.WithValue(ctx, loggerOptKey, loggerOpt)
+}
+
+// GetLogger attempts to retrieve LoggerFn from context, returns it if found, otherwise returns no-op function
+func GetLogger(ctx context.Context) (LoggerFn, bool) {
+ logger, ok := ctx.Value(loggerKey).(LoggerFn)
+ if !ok {
+ logger = func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {}
+ }
+ return logger, ok
+}
+
+// GetLoggerOpt attempts to retrieve LoggerOpt from context, returns it if found, otherwise returns NewLoggerOpt()
+func GetLoggerOpt(ctx context.Context) LoggerOpt {
+ loggerOpt, ok := ctx.Value(loggerOptKey).(LoggerOpt)
+ if ok {
+ return loggerOpt
+ }
+ return NewLoggerOpt()
+}
+
+// WithSyncLogger starts a new logger with the options passed in and saves it to ctx for retrieval later
+func WithSyncLogger(ctx context.Context, opt LoggerOpt) context.Context {
+ ctx = WithLoggerOpt(ctx, opt)
+ return WithLogger(ctx, func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {
+ if opt.LoggerFn != nil {
+ opt.LoggerFn(ctx, sigil, src, dst, err)
+ } else {
+ SyncFprintf(opt.Combined, "%c %s\n", sigil, dst.Remote())
+ }
+ })
+}
+
+// NewLoggerOpt returns a new LoggerOpt struct with defaults
+func NewLoggerOpt() LoggerOpt {
+ opt := LoggerOpt{
+ Combined: new(bytes.Buffer),
+ MissingOnSrc: new(bytes.Buffer),
+ MissingOnDst: new(bytes.Buffer),
+ Match: new(bytes.Buffer),
+ Differ: new(bytes.Buffer),
+ Error: new(bytes.Buffer),
+ DestAfter: new(bytes.Buffer),
+ JSON: new(bytes.Buffer),
+ }
+ return opt
+}
+
+// Winner predicts which side (src or dst) should end up winning out on the dst.
+type Winner struct {
+ Obj fs.DirEntry // the object that should exist on dst post-sync, if any
+ Side string // whether the winning object was from the src or dst
+ Err error // whether there's an error preventing us from predicting winner correctly (not whether there was a sync error more generally)
+}
+
+// WinningSide can be called in a LoggerFn to predict what the dest will look like post-sync
+//
+// This attempts to account for every case in which dst (intentionally) does not match src after a sync.
+//
+// Known issues / cases we can't confidently predict yet:
+//
+// --max-duration / CutoffModeHard
+// --compare-dest / --copy-dest (because equal() is called multiple times for the same file)
+// server-side moves of an entire dir at once (because we never get the individual file objects in the dir)
+// High-level retries, because there would be dupes (use --retries 1 to disable)
+// Possibly some error scenarios
+func WinningSide(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) Winner {
+ winner := Winner{nil, "none", nil}
+ opt := GetLoggerOpt(ctx)
+ ci := fs.GetConfig(ctx)
+
+ if err == fs.ErrorIsDir {
+ winner.Err = err
+ if sigil == MissingOnSrc {
+ if (opt.DeleteModeOff || ci.DryRun) && dst != nil {
+ winner.Obj = dst
+ winner.Side = "dst" // whatever's on dst will remain so after DryRun
+ return winner
+ }
+ return winner // none, because dst should just get deleted
+ }
+ if sigil == MissingOnDst && ci.DryRun {
+ return winner // none, because it does not currently exist on dst, and will still not exist after DryRun
+ } else if ci.DryRun && dst != nil {
+ winner.Obj = dst
+ winner.Side = "dst"
+ } else if src != nil {
+ winner.Obj = src
+ winner.Side = "src"
+ }
+ return winner
+ }
+
+ _, srcOk := src.(fs.Object)
+ _, dstOk := dst.(fs.Object)
+ if !srcOk && !dstOk {
+ return winner // none, because we don't have enough info to continue.
+ }
+
+ switch sigil {
+ case MissingOnSrc:
+ if opt.DeleteModeOff || ci.DryRun { // i.e. it's a copy, not sync (or it's a DryRun)
+ winner.Obj = dst
+ winner.Side = "dst" // whatever's on dst will remain so after DryRun
+ return winner
+ }
+ return winner // none, because dst should just get deleted
+ case Match, Differ, MissingOnDst:
+ if sigil == MissingOnDst && ci.DryRun {
+ return winner // none, because it does not currently exist on dst, and will still not exist after DryRun
+ }
+ winner.Obj = src
+ winner.Side = "src" // presume dst will end up matching src unless changed below
+ if sigil == Match && (ci.SizeOnly || ci.CheckSum || ci.IgnoreSize || ci.UpdateOlder || ci.NoUpdateModTime) {
+ winner.Obj = dst
+ winner.Side = "dst" // ignore any differences with src because of user flags
+ }
+ if ci.IgnoreTimes {
+ winner.Obj = src
+ winner.Side = "src" // copy src to dst unconditionally
+ }
+ if (sigil == Match || sigil == Differ) && (ci.IgnoreExisting || ci.Immutable) {
+ winner.Obj = dst
+ winner.Side = "dst" // dst should remain unchanged if it already exists (and we know it does because it's Match or Differ)
+ }
+ if ci.DryRun {
+ winner.Obj = dst
+ winner.Side = "dst" // dst should remain unchanged after DryRun (note that we handled MissingOnDst earlier)
+ }
+ return winner
+ case TransferError:
+ winner.Obj = dst
+ winner.Side = "dst" // usually, dst should not change if there's an error
+ if dst == nil {
+ winner.Obj = src
+ winner.Side = "src" // but if for some reason we have a src and not a dst, go with it
+ }
+ if winner.Obj != nil {
+ if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, errors.New("max transfer duration reached as set by --max-duration")) {
+ winner.Err = err // we can't confidently predict what survives if CutoffModeHard
+ }
+ return winner // we know at least one of the objects
+ }
+ }
+ // should only make it this far if it's TransferError and both src and dst are nil
+ winner.Side = "none"
+ winner.Err = fmt.Errorf("unknown case -- can't determine winner. %v", err)
+ fs.Debugf(winner.Obj, "%v", winner.Err)
+ return winner
+}
+
+// SetListFormat sets opt.ListFormat for destAfter
+// TODO: possibly refactor duplicate code from cmd/lsf, where this is mostly copied from
+func (opt *LoggerOpt) SetListFormat(ctx context.Context, cmdFlags *pflag.FlagSet) {
+ // Work out if the separatorFlag was supplied or not
+ separatorFlag := cmdFlags.Lookup("separator")
+ separatorFlagSupplied := separatorFlag != nil && separatorFlag.Changed
+ // Default the separator to , if using CSV
+ if opt.Csv && !separatorFlagSupplied {
+ opt.Separator = ","
+ }
+
+ var list ListFormat
+ list.SetSeparator(opt.Separator)
+ list.SetCSV(opt.Csv)
+ list.SetDirSlash(opt.DirSlash)
+ list.SetAbsolute(opt.Absolute)
+ var JSONOpt = ListJSONOpt{
+ NoModTime: true,
+ NoMimeType: true,
+ DirsOnly: opt.DirsOnly,
+ FilesOnly: opt.FilesOnly,
+ // Recurse: opt.Recurse,
+ }
+
+ for _, char := range opt.Format {
+ switch char {
+ case 'p':
+ list.AddPath()
+ case 't':
+ list.AddModTime(opt.TimeFormat)
+ JSONOpt.NoModTime = false
+ case 's':
+ list.AddSize()
+ case 'h':
+ list.AddHash(opt.HashType)
+ JSONOpt.ShowHash = true
+ JSONOpt.HashTypes = []string{opt.HashType.String()}
+ case 'i':
+ list.AddID()
+ case 'm':
+ list.AddMimeType()
+ JSONOpt.NoMimeType = false
+ case 'e':
+ list.AddEncrypted()
+ JSONOpt.ShowEncrypted = true
+ case 'o':
+ list.AddOrigID()
+ JSONOpt.ShowOrigIDs = true
+ case 'T':
+ list.AddTier()
+ case 'M':
+ list.AddMetadata()
+ JSONOpt.Metadata = true
+ default:
+ fs.Errorf(nil, "unknown format character %q", char)
+ }
+ }
+ opt.ListFormat = list
+ opt.JSONOpt = JSONOpt
+}
+
+// NewListJSON makes a new *listJSON for destAfter
+func (opt *LoggerOpt) NewListJSON(ctx context.Context, fdst fs.Fs, remote string) {
+ opt.LJ, _ = newListJSON(ctx, fdst, remote, &opt.JSONOpt)
+ //fs.Debugf(nil, "%v", opt.LJ)
+}
+
+// JSONEntry returns a *ListJSONItem for destAfter
+func (opt *LoggerOpt) JSONEntry(ctx context.Context, entry fs.DirEntry) (*ListJSONItem, error) {
+ return opt.LJ.entry(ctx, entry)
+}
+
+// PrintDestAfter writes a *ListJSONItem to opt.DestAfter
+func (opt *LoggerOpt) PrintDestAfter(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {
+ entry := WinningSide(ctx, sigil, src, dst, err)
+ if entry.Obj != nil {
+ JSONEntry, _ := opt.JSONEntry(ctx, entry.Obj)
+ _, _ = fmt.Fprintln(opt.DestAfter, opt.ListFormat.Format(JSONEntry))
+ }
+}
generated by cgit v1.0 at 2026-05-27 06:04:39 +0000