aboutsummaryrefslogtreecommitdiff
path: root/fstest/fstest.go
diff options
context:
space:
mode:
Diffstat (limited to 'fstest/fstest.go')
-rw-r--r--fstest/fstest.go651
1 files changed, 651 insertions, 0 deletions
diff --git a/fstest/fstest.go b/fstest/fstest.go
new file mode 100644
index 0000000..a0205a1
--- /dev/null
+++ b/fstest/fstest.go
@@ -0,0 +1,651 @@
+// Package fstest provides utilities for testing the Fs
+package fstest
+
+// FIXME put name of test FS in Fs structure
+
+import (
+ "bytes"
+ "compress/gzip"
+ "context"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "path"
+ "path/filepath"
+ "regexp"
+ "runtime"
+ "sort"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/config"
+ "github.com/rclone/rclone/fs/config/configfile"
+ "github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/log"
+ "github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/fstest/testy"
+ "github.com/rclone/rclone/lib/random"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "golang.org/x/text/unicode/norm"
+)
+
+// Globals
+var (
+ RemoteName = flag.String("remote", "", "Remote to test with, defaults to local filesystem")
+ Verbose = flag.Bool("verbose", false, "Set to enable logging")
+ DumpHeaders = flag.Bool("dump-headers", false, "Set to dump headers (needs -verbose)")
+ DumpBodies = flag.Bool("dump-bodies", false, "Set to dump bodies (needs -verbose)")
+ Individual = flag.Bool("individual", false, "Make individual bucket/container/directory for each test - much slower")
+ LowLevelRetries = flag.Int("low-level-retries", 10, "Number of low level retries")
+ UseListR = flag.Bool("fast-list", false, "Use recursive list if available. Uses more memory but fewer transactions.")
+ // SizeLimit signals tests to skip maximum test file size and skip inappropriate runs
+ SizeLimit = flag.Int64("size-limit", 0, "Limit maximum test file size")
+ // ListRetries is the number of times to retry a listing to overcome eventual consistency
+ ListRetries = flag.Int("list-retries", 3, "Number or times to retry listing")
+ // MatchTestRemote matches the remote names used for testing
+ MatchTestRemote = regexp.MustCompile(`^rclone-test-[abcdefghijklmnopqrstuvwxyz0123456789]{12}$`)
+)
+
+// Initialise rclone for testing
+func Initialise() {
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
+ // Never ask for passwords, fail instead.
+ // If your local config is encrypted set environment variable
+ // "RCLONE_CONFIG_PASS=hunter2" (or your password)
+ ci.AskPassword = false
+ // Override the config file from the environment - we don't
+ // parse the flags any more so this doesn't happen
+ // automatically
+ if envConfig := os.Getenv("RCLONE_CONFIG"); envConfig != "" {
+ _ = config.SetConfigPath(envConfig)
+ }
+ if *RemoteName == "local" {
+ *RemoteName = ""
+ }
+ configfile.Install()
+ accounting.Start(ctx)
+ if *Verbose {
+ ci.LogLevel = fs.LogLevelDebug
+ }
+ if *DumpHeaders {
+ ci.Dump |= fs.DumpHeaders
+ }
+ if *DumpBodies {
+ ci.Dump |= fs.DumpBodies
+ }
+ ci.LowLevelRetries = *LowLevelRetries
+ ci.UseListR = *UseListR
+ log.InitLogging()
+ _ = fs.LogReload(ci)
+}
+
+// Item represents an item for checking
+type Item struct {
+ Path string
+ Hashes map[hash.Type]string
+ ModTime time.Time
+ Size int64
+}
+
+// NewItem creates an item from a string content
+func NewItem(Path, Content string, modTime time.Time) Item {
+ i := Item{
+ Path: Path,
+ ModTime: modTime,
+ Size: int64(len(Content)),
+ }
+ hash := hash.NewMultiHasher()
+ buf := bytes.NewBufferString(Content)
+ _, err := io.Copy(hash, buf)
+ if err != nil {
+ fs.Fatalf(nil, "Failed to create item: %v", err)
+ }
+ i.Hashes = hash.Sums()
+ return i
+}
+
+// CheckTimeEqualWithPrecision checks the times are equal within the
+// precision, returns the delta and a flag
+func CheckTimeEqualWithPrecision(t0, t1 time.Time, precision time.Duration) (time.Duration, bool) {
+ dt := t0.Sub(t1)
+ if dt >= precision || dt <= -precision {
+ return dt, false
+ }
+ return dt, true
+}
+
+// AssertTimeEqualWithPrecision checks that want is within precision
+// of got, asserting that with t and logging remote
+func AssertTimeEqualWithPrecision(t *testing.T, remote string, want, got time.Time, precision time.Duration) {
+ dt, ok := CheckTimeEqualWithPrecision(want, got, precision)
+ assert.True(t, ok, fmt.Sprintf("%s: Modification time difference too big |%s| > %s (want %s vs got %s) (precision %s)", remote, dt, precision, want, got, precision))
+}
+
+// CheckModTime checks the mod time to the given precision
+func (i *Item) CheckModTime(t *testing.T, obj fs.Object, modTime time.Time, precision time.Duration) {
+ AssertTimeEqualWithPrecision(t, obj.Remote(), i.ModTime, modTime, precision)
+}
+
+// CheckHashes checks all the hashes the object supports are correct
+func (i *Item) CheckHashes(t *testing.T, obj fs.Object) {
+ require.NotNil(t, obj)
+ types := obj.Fs().Hashes().Array()
+ for _, Hash := range types {
+ // Check attributes
+ sum, err := obj.Hash(context.Background(), Hash)
+ require.NoError(t, err)
+ assert.True(t, hash.Equals(i.Hashes[Hash], sum), fmt.Sprintf("%s/%s: %v hash incorrect - expecting %q got %q", obj.Fs().String(), obj.Remote(), Hash, i.Hashes[Hash], sum))
+ }
+}
+
+// Check checks all the attributes of the object are correct
+func (i *Item) Check(t *testing.T, obj fs.Object, precision time.Duration) {
+ i.CheckHashes(t, obj)
+ assert.Equal(t, i.Size, obj.Size(), fmt.Sprintf("%s: size incorrect file=%d vs obj=%d", i.Path, i.Size, obj.Size()))
+ i.CheckModTime(t, obj, obj.ModTime(context.Background()), precision)
+}
+
+// Normalize runs a utf8 normalization on the string if running on OS
+// X. This is because OS X denormalizes file names it writes to the
+// local file system.
+func Normalize(name string) string {
+ if runtime.GOOS == "darwin" {
+ name = norm.NFC.String(name)
+ }
+ return name
+}
+
+// Items represents all items for checking
+type Items struct {
+ byName map[string]*Item
+ byNameAlt map[string]*Item
+ items []Item
+}
+
+// NewItems makes an Items
+func NewItems(items []Item) *Items {
+ is := &Items{
+ byName: make(map[string]*Item),
+ byNameAlt: make(map[string]*Item),
+ items: items,
+ }
+ // Fill up byName
+ for i := range items {
+ is.byName[Normalize(items[i].Path)] = &items[i]
+ }
+ return is
+}
+
+// Find checks off an item
+func (is *Items) Find(t *testing.T, obj fs.Object, precision time.Duration) {
+ remote := Normalize(obj.Remote())
+ i, ok := is.byName[remote]
+ if !ok {
+ i, ok = is.byNameAlt[remote]
+ assert.True(t, ok, fmt.Sprintf("Unexpected file %q", remote))
+ }
+ if i != nil {
+ delete(is.byName, i.Path)
+ i.Check(t, obj, precision)
+ }
+}
+
+// Done checks all finished
+func (is *Items) Done(t *testing.T) {
+ if len(is.byName) != 0 {
+ for name := range is.byName {
+ t.Logf("Not found %q", name)
+ }
+ }
+ assert.Equal(t, 0, len(is.byName), fmt.Sprintf("%d objects not found", len(is.byName)))
+}
+
+// makeListingFromItems returns a string representation of the items
+//
+// it returns two possible strings, one normal and one for windows
+func makeListingFromItems(items []Item) string {
+ nameLengths := make([]string, len(items))
+ for i, item := range items {
+ remote := Normalize(item.Path)
+ nameLengths[i] = fmt.Sprintf("%s (%d)", remote, item.Size)
+ }
+ sort.Strings(nameLengths)
+ return strings.Join(nameLengths, ", ")
+}
+
+// makeListingFromObjects returns a string representation of the objects
+func makeListingFromObjects(objs []fs.Object) string {
+ nameLengths := make([]string, len(objs))
+ for i, obj := range objs {
+ nameLengths[i] = fmt.Sprintf("%s (%d)", Normalize(obj.Remote()), obj.Size())
+ }
+ sort.Strings(nameLengths)
+ return strings.Join(nameLengths, ", ")
+}
+
+// filterEmptyDirs removes any empty (or containing only directories)
+// directories from expectedDirs
+func filterEmptyDirs(t *testing.T, items []Item, expectedDirs []string) (newExpectedDirs []string) {
+ dirs := map[string]struct{}{"": {}}
+ for _, item := range items {
+ base := item.Path
+ for {
+ base = path.Dir(base)
+ if base == "." || base == "/" {
+ break
+ }
+ dirs[base] = struct{}{}
+ }
+ }
+ for _, expectedDir := range expectedDirs {
+ if _, found := dirs[expectedDir]; found {
+ newExpectedDirs = append(newExpectedDirs, expectedDir)
+ } else {
+ t.Logf("Filtering empty directory %q", expectedDir)
+ }
+ }
+ return newExpectedDirs
+}
+
+// CheckListingWithRoot checks the fs to see if it has the
+// expected contents with the given precision.
+//
+// If expectedDirs is non nil then we check those too. Note that no
+// directories returned is also OK as some remotes don't return
+// directories.
+//
+// dir is the directory used for the listing.
+func CheckListingWithRoot(t *testing.T, f fs.Fs, dir string, items []Item, expectedDirs []string, precision time.Duration) {
+ if expectedDirs != nil && !f.Features().CanHaveEmptyDirectories {
+ expectedDirs = filterEmptyDirs(t, items, expectedDirs)
+ }
+ is := NewItems(items)
+ ctx := context.Background()
+ oldErrors := accounting.Stats(ctx).GetErrors()
+ var objs []fs.Object
+ var dirs []fs.Directory
+ var err error
+ retries := *ListRetries
+ sleep := time.Second / 2
+ wantListing := makeListingFromItems(items)
+ gotListing := "<unset>"
+ listingOK := false
+ for i := 1; i <= retries; i++ {
+ objs, dirs, err = walk.GetAll(ctx, f, dir, true, -1)
+ if err != nil && err != fs.ErrorDirNotFound {
+ t.Fatalf("Error listing: %v", err)
+ }
+ gotListing = makeListingFromObjects(objs)
+
+ listingOK = wantListing == gotListing
+ if listingOK && (expectedDirs == nil || len(dirs) == len(expectedDirs)) {
+ // Put an extra sleep in if we did any retries just to make sure it really
+ // is consistent
+ if i != 1 {
+ extraSleep := 5*time.Second + sleep
+ t.Logf("Sleeping for %v just to make sure", extraSleep)
+ time.Sleep(extraSleep)
+ }
+ break
+ }
+ sleep *= 2
+ t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries)
+ time.Sleep(sleep)
+ if doDirCacheFlush := f.Features().DirCacheFlush; doDirCacheFlush != nil {
+ t.Logf("Flushing the directory cache")
+ doDirCacheFlush()
+ }
+ }
+ assert.True(t, listingOK, fmt.Sprintf("listing wrong, want\n %s got\n %s", wantListing, gotListing))
+ for _, obj := range objs {
+ require.NotNil(t, obj)
+ is.Find(t, obj, precision)
+ }
+ is.Done(t)
+ // Don't notice an error when listing an empty directory
+ if len(items) == 0 && oldErrors == 0 && accounting.Stats(ctx).GetErrors() == 1 {
+ accounting.Stats(ctx).ResetErrors()
+ }
+ // Check the directories
+ if expectedDirs != nil {
+ expectedDirsCopy := make([]string, len(expectedDirs))
+ for i, dir := range expectedDirs {
+ expectedDirsCopy[i] = Normalize(dir)
+ }
+ actualDirs := []string{}
+ for _, dir := range dirs {
+ actualDirs = append(actualDirs, Normalize(dir.Remote()))
+ }
+ sort.Strings(actualDirs)
+ sort.Strings(expectedDirsCopy)
+ assert.Equal(t, expectedDirsCopy, actualDirs, "directories")
+ }
+}
+
+// CheckListingWithPrecision checks the fs to see if it has the
+// expected contents with the given precision.
+//
+// If expectedDirs is non nil then we check those too. Note that no
+// directories returned is also OK as some remotes don't return
+// directories.
+func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, expectedDirs []string, precision time.Duration) {
+ CheckListingWithRoot(t, f, "", items, expectedDirs, precision)
+}
+
+// CheckListing checks the fs to see if it has the expected contents
+func CheckListing(t *testing.T, f fs.Fs, items []Item) {
+ precision := f.Precision()
+ CheckListingWithPrecision(t, f, items, nil, precision)
+}
+
+// CheckItemsWithPrecision checks the fs with the specified precision
+// to see if it has the expected items.
+func CheckItemsWithPrecision(t *testing.T, f fs.Fs, precision time.Duration, items ...Item) {
+ CheckListingWithPrecision(t, f, items, nil, precision)
+}
+
+// CheckItems checks the fs to see if it has only the items passed in
+// using a precision of fs.Config.ModifyWindow
+func CheckItems(t *testing.T, f fs.Fs, items ...Item) {
+ CheckListingWithPrecision(t, f, items, nil, fs.GetModifyWindow(context.TODO(), f))
+}
+
+// CompareItems compares a set of DirEntries to a slice of items and a list of dirs
+// The modtimes are compared with the precision supplied
+func CompareItems(t *testing.T, entries fs.DirEntries, items []Item, expectedDirs []string, precision time.Duration, what string) {
+ is := NewItems(items)
+ var objs []fs.Object
+ var dirs []fs.Directory
+ wantListing := makeListingFromItems(items)
+ for _, entry := range entries {
+ switch x := entry.(type) {
+ case fs.Directory:
+ dirs = append(dirs, x)
+ case fs.Object:
+ objs = append(objs, x)
+ // do nothing
+ default:
+ t.Fatalf("unknown object type %T", entry)
+ }
+ }
+
+ gotListing := makeListingFromObjects(objs)
+ listingOK := wantListing == gotListing
+ assert.True(t, listingOK, fmt.Sprintf("%s not equal, want\n %s got\n %s", what, wantListing, gotListing))
+ for _, obj := range objs {
+ require.NotNil(t, obj)
+ is.Find(t, obj, precision)
+ }
+ is.Done(t)
+ // Check the directories
+ if expectedDirs != nil {
+ expectedDirsCopy := make([]string, len(expectedDirs))
+ for i, dir := range expectedDirs {
+ expectedDirsCopy[i] = Normalize(dir)
+ }
+ actualDirs := []string{}
+ for _, dir := range dirs {
+ actualDirs = append(actualDirs, Normalize(dir.Remote()))
+ }
+ sort.Strings(actualDirs)
+ sort.Strings(expectedDirsCopy)
+ assert.Equal(t, expectedDirsCopy, actualDirs, "directories not equal")
+ }
+}
+
+// Time parses a time string or logs a fatal error
+func Time(timeString string) time.Time {
+ t, err := time.Parse(time.RFC3339Nano, timeString)
+ if err != nil {
+ fs.Fatalf(nil, "Failed to parse time %q: %v", timeString, err)
+ }
+ return t
+}
+
+// LocalRemote creates a temporary directory name for local remotes
+func LocalRemote() (path string, err error) {
+ path, err = os.MkdirTemp("", "rclone")
+ if err == nil {
+ // Now remove the directory
+ err = os.Remove(path)
+ }
+ path = filepath.ToSlash(path)
+ return
+}
+
+// RandomRemoteName makes a random bucket or subdirectory name
+//
+// Returns a random remote name plus the leaf name
+func RandomRemoteName(remoteName string) (string, string, error) {
+ var err error
+ var leafName string
+
+ // Make a directory if remote name is null
+ if remoteName == "" {
+ remoteName, err = LocalRemote()
+ if err != nil {
+ return "", "", err
+ }
+ } else {
+ if !strings.HasSuffix(remoteName, ":") {
+ remoteName += "/"
+ }
+ leafName = "rclone-test-" + random.String(12)
+ if !MatchTestRemote.MatchString(leafName) {
+ fs.Fatalf(nil, "%q didn't match the test remote name regexp", leafName)
+ }
+ remoteName += leafName
+ }
+ return remoteName, leafName, nil
+}
+
+// RandomRemote makes a random bucket or subdirectory on the remote
+// from the -remote parameter
+//
+// Call the finalise function returned to Purge the fs at the end (and
+// the parent if necessary)
+//
+// Returns the remote, its url, a finaliser and an error
+func RandomRemote() (fs.Fs, string, func(), error) {
+ var err error
+ var parentRemote fs.Fs
+ remoteName := *RemoteName
+
+ remoteName, _, err = RandomRemoteName(remoteName)
+ if err != nil {
+ return nil, "", nil, err
+ }
+
+ remote, err := fs.NewFs(context.Background(), remoteName)
+ if err != nil {
+ return nil, "", nil, err
+ }
+
+ finalise := func() {
+ Purge(remote)
+ if parentRemote != nil {
+ Purge(parentRemote)
+ if err != nil {
+ fs.Logf(nil, "Failed to purge %v: %v", parentRemote, err)
+ }
+ }
+ }
+
+ return remote, remoteName, finalise, nil
+}
+
+// Purge is a simplified re-implementation of operations.Purge for the
+// test routine cleanup to avoid circular dependencies.
+//
+// It logs errors rather than returning them
+func Purge(f fs.Fs) {
+ // Create a stats group here so errors in the cleanup don't
+ // interfere with the global stats.
+ ctx := accounting.WithStatsGroup(context.Background(), "test-cleanup")
+ var err error
+ doFallbackPurge := true
+ if doPurge := f.Features().Purge; doPurge != nil {
+ doFallbackPurge = false
+ fs.Debugf(f, "Purge remote")
+ err = doPurge(ctx, "")
+ if err == fs.ErrorCantPurge {
+ doFallbackPurge = true
+ }
+ }
+ if doFallbackPurge {
+ dirs := []string{""}
+ err = walk.ListR(ctx, f, "", true, -1, walk.ListAll, func(entries fs.DirEntries) error {
+ var err error
+ entries.ForObject(func(obj fs.Object) {
+ fs.Debugf(f, "Purge object %q", obj.Remote())
+ err = obj.Remove(ctx)
+ if err != nil {
+ fs.Logf(nil, "purge failed to remove %q: %v", obj.Remote(), err)
+ }
+ })
+ entries.ForDir(func(dir fs.Directory) {
+ dirs = append(dirs, dir.Remote())
+ })
+ return nil
+ })
+ sort.Strings(dirs)
+ for i := len(dirs) - 1; i >= 0; i-- {
+ dir := dirs[i]
+ fs.Debugf(f, "Purge dir %q", dir)
+ err := f.Rmdir(ctx, dir)
+ if err != nil {
+ fs.Logf(nil, "purge failed to rmdir %q: %v", dir, err)
+ }
+ }
+ }
+ if err != nil {
+ fs.Logf(nil, "purge failed: %v", err)
+ }
+}
+
+// NewObject finds the object on the remote
+func NewObject(ctx context.Context, t *testing.T, f fs.Fs, remote string) fs.Object {
+ var obj fs.Object
+ var err error
+ sleepTime := 1 * time.Second
+ for i := 1; i <= *ListRetries; i++ {
+ obj, err = f.NewObject(ctx, remote)
+ if err == nil {
+ break
+ }
+ t.Logf("Sleeping for %v for findObject eventual consistency: %d/%d (%v)", sleepTime, i, *ListRetries, err)
+ time.Sleep(sleepTime)
+ sleepTime = (sleepTime * 3) / 2
+ }
+ require.NoError(t, err)
+ return obj
+}
+
+// NewDirectoryRetries finds the directory with remote in f
+//
+// If directory can't be found it returns an error wrapping fs.ErrorDirNotFound
+//
+// One day this will be an rclone primitive
+func NewDirectoryRetries(ctx context.Context, t *testing.T, f fs.Fs, remote string, retries int) (fs.Directory, error) {
+ var err error
+ var dir fs.Directory
+ sleepTime := 1 * time.Second
+ root := path.Dir(remote)
+ if root == "." {
+ root = ""
+ }
+ for i := 1; i <= retries; i++ {
+ var entries fs.DirEntries
+ entries, err = f.List(ctx, root)
+ if err != nil {
+ continue
+ }
+ for _, entry := range entries {
+ var ok bool
+ dir, ok = entry.(fs.Directory)
+ if ok && dir.Remote() == remote {
+ return dir, nil
+ }
+ }
+ err = fmt.Errorf("directory %q not found in %q: %w", remote, root, fs.ErrorDirNotFound)
+ if i < retries {
+ t.Logf("Sleeping for %v for NewDirectoryRetries eventual consistency: %d/%d (%v)", sleepTime, i, retries, err)
+ time.Sleep(sleepTime)
+ sleepTime = (sleepTime * 3) / 2
+ }
+ }
+ return dir, err
+}
+
+// NewDirectory finds the directory with remote in f
+//
+// One day this will be an rclone primitive
+func NewDirectory(ctx context.Context, t *testing.T, f fs.Fs, remote string) fs.Directory {
+ dir, err := NewDirectoryRetries(ctx, t, f, remote, *ListRetries)
+ require.NoError(t, err)
+ return dir
+}
+
+// CheckEntryMetadata checks the metadata on the directory
+//
+// This checks a limited set of metadata on the directory
+func CheckEntryMetadata(ctx context.Context, t *testing.T, f fs.Fs, entry fs.DirEntry, wantMeta fs.Metadata) {
+ features := f.Features()
+ do, ok := entry.(fs.Metadataer)
+ require.True(t, ok, "Didn't find expected Metadata() method on %T", entry)
+ gotMeta, err := do.Metadata(ctx)
+ require.NoError(t, err)
+
+ for k, v := range wantMeta {
+ switch k {
+ case "mtime", "atime", "btime", "ctime":
+ // Check the system time Metadata
+ wantT, err := time.Parse(time.RFC3339, v)
+ require.NoError(t, err)
+ gotT, err := time.Parse(time.RFC3339, gotMeta[k])
+ require.NoError(t, err)
+ AssertTimeEqualWithPrecision(t, entry.Remote(), wantT, gotT, f.Precision())
+ default:
+ // Check the User metadata if we can
+ _, isDir := entry.(fs.Directory)
+ if (isDir && features.UserDirMetadata) || (!isDir && features.UserMetadata) {
+ assert.Equal(t, v, gotMeta[k])
+ }
+ }
+ }
+}
+
+// CheckDirModTime checks the modtime on the directory
+func CheckDirModTime(ctx context.Context, t *testing.T, f fs.Fs, dir fs.Directory, wantT time.Time) {
+ if f.Features().DirSetModTime == nil && f.Features().MkdirMetadata == nil {
+ fs.Debugf(f, "Skipping modtime test as remote does not support DirSetModTime or MkdirMetadata")
+ return
+ }
+ gotT := dir.ModTime(ctx)
+ precision := f.Precision()
+ // For unknown reasons the precision of modification times of
+ // directories on the CI is about >15mS. The tests work fine
+ // when run in Virtualbox though so I conjecture this is
+ // something to do with the file system used there.
+ if runtime.GOOS == "windows" && testy.CI() {
+ precision = 100 * time.Millisecond
+ }
+ AssertTimeEqualWithPrecision(t, dir.Remote(), wantT, gotT, precision)
+}
+
+// Gz returns a compressed version of its input string
+func Gz(t *testing.T, s string) string {
+ var buf bytes.Buffer
+ zw := gzip.NewWriter(&buf)
+ _, err := zw.Write([]byte(s))
+ require.NoError(t, err)
+ err = zw.Close()
+ require.NoError(t, err)
+ return buf.String()
+}
generated by cgit v1.0 at 2026-05-27 04:40:40 +0000