diff options
63 files changed, 8256 insertions, 0 deletions
diff --git a/backend/studip/studip_test.go b/backend/studip/studip_test.go new file mode 100644 index 0000000..d851424 --- /dev/null +++ b/backend/studip/studip_test.go @@ -0,0 +1,16 @@ +package studip_test + +import ( + "testing" + + "github.com/mewsen/rclone-studip-backend-oot/backend/studip" + "github.com/mewsen/rclone-studip-backend-oot/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestStudIP:rclone", + NilObject: (*studip.Object)(nil), + }) +} diff --git a/fstest/fstest.go b/fstest/fstest.go new file mode 100644 index 0000000..a0205a1 --- /dev/null +++ b/fstest/fstest.go @@ -0,0 +1,651 @@ +// Package fstest provides utilities for testing the Fs +package fstest + +// FIXME put name of test FS in Fs structure + +import ( + "bytes" + "compress/gzip" + "context" + "flag" + "fmt" + "io" + "os" + "path" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "testing" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/config/configfile" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/log" + "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fstest/testy" + "github.com/rclone/rclone/lib/random" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/text/unicode/norm" +) + +// Globals +var ( + RemoteName = flag.String("remote", "", "Remote to test with, defaults to local filesystem") + Verbose = flag.Bool("verbose", false, "Set to enable logging") + DumpHeaders = flag.Bool("dump-headers", false, "Set to dump headers (needs -verbose)") + DumpBodies = flag.Bool("dump-bodies", false, "Set to dump bodies (needs -verbose)") + Individual = flag.Bool("individual", false, "Make individual bucket/container/directory for each test - much slower") + LowLevelRetries = flag.Int("low-level-retries", 10, "Number of low level retries") + UseListR = flag.Bool("fast-list", false, "Use recursive list if available. Uses more memory but fewer transactions.") + // SizeLimit signals tests to skip maximum test file size and skip inappropriate runs + SizeLimit = flag.Int64("size-limit", 0, "Limit maximum test file size") + // ListRetries is the number of times to retry a listing to overcome eventual consistency + ListRetries = flag.Int("list-retries", 3, "Number or times to retry listing") + // MatchTestRemote matches the remote names used for testing + MatchTestRemote = regexp.MustCompile(`^rclone-test-[abcdefghijklmnopqrstuvwxyz0123456789]{12}$`) +) + +// Initialise rclone for testing +func Initialise() { + ctx := context.Background() + ci := fs.GetConfig(ctx) + // Never ask for passwords, fail instead. + // If your local config is encrypted set environment variable + // "RCLONE_CONFIG_PASS=hunter2" (or your password) + ci.AskPassword = false + // Override the config file from the environment - we don't + // parse the flags any more so this doesn't happen + // automatically + if envConfig := os.Getenv("RCLONE_CONFIG"); envConfig != "" { + _ = config.SetConfigPath(envConfig) + } + if *RemoteName == "local" { + *RemoteName = "" + } + configfile.Install() + accounting.Start(ctx) + if *Verbose { + ci.LogLevel = fs.LogLevelDebug + } + if *DumpHeaders { + ci.Dump |= fs.DumpHeaders + } + if *DumpBodies { + ci.Dump |= fs.DumpBodies + } + ci.LowLevelRetries = *LowLevelRetries + ci.UseListR = *UseListR + log.InitLogging() + _ = fs.LogReload(ci) +} + +// Item represents an item for checking +type Item struct { + Path string + Hashes map[hash.Type]string + ModTime time.Time + Size int64 +} + +// NewItem creates an item from a string content +func NewItem(Path, Content string, modTime time.Time) Item { + i := Item{ + Path: Path, + ModTime: modTime, + Size: int64(len(Content)), + } + hash := hash.NewMultiHasher() + buf := bytes.NewBufferString(Content) + _, err := io.Copy(hash, buf) + if err != nil { + fs.Fatalf(nil, "Failed to create item: %v", err) + } + i.Hashes = hash.Sums() + return i +} + +// CheckTimeEqualWithPrecision checks the times are equal within the +// precision, returns the delta and a flag +func CheckTimeEqualWithPrecision(t0, t1 time.Time, precision time.Duration) (time.Duration, bool) { + dt := t0.Sub(t1) + if dt >= precision || dt <= -precision { + return dt, false + } + return dt, true +} + +// AssertTimeEqualWithPrecision checks that want is within precision +// of got, asserting that with t and logging remote +func AssertTimeEqualWithPrecision(t *testing.T, remote string, want, got time.Time, precision time.Duration) { + dt, ok := CheckTimeEqualWithPrecision(want, got, precision) + assert.True(t, ok, fmt.Sprintf("%s: Modification time difference too big |%s| > %s (want %s vs got %s) (precision %s)", remote, dt, precision, want, got, precision)) +} + +// CheckModTime checks the mod time to the given precision +func (i *Item) CheckModTime(t *testing.T, obj fs.Object, modTime time.Time, precision time.Duration) { + AssertTimeEqualWithPrecision(t, obj.Remote(), i.ModTime, modTime, precision) +} + +// CheckHashes checks all the hashes the object supports are correct +func (i *Item) CheckHashes(t *testing.T, obj fs.Object) { + require.NotNil(t, obj) + types := obj.Fs().Hashes().Array() + for _, Hash := range types { + // Check attributes + sum, err := obj.Hash(context.Background(), Hash) + require.NoError(t, err) + assert.True(t, hash.Equals(i.Hashes[Hash], sum), fmt.Sprintf("%s/%s: %v hash incorrect - expecting %q got %q", obj.Fs().String(), obj.Remote(), Hash, i.Hashes[Hash], sum)) + } +} + +// Check checks all the attributes of the object are correct +func (i *Item) Check(t *testing.T, obj fs.Object, precision time.Duration) { + i.CheckHashes(t, obj) + assert.Equal(t, i.Size, obj.Size(), fmt.Sprintf("%s: size incorrect file=%d vs obj=%d", i.Path, i.Size, obj.Size())) + i.CheckModTime(t, obj, obj.ModTime(context.Background()), precision) +} + +// Normalize runs a utf8 normalization on the string if running on OS +// X. This is because OS X denormalizes file names it writes to the +// local file system. +func Normalize(name string) string { + if runtime.GOOS == "darwin" { + name = norm.NFC.String(name) + } + return name +} + +// Items represents all items for checking +type Items struct { + byName map[string]*Item + byNameAlt map[string]*Item + items []Item +} + +// NewItems makes an Items +func NewItems(items []Item) *Items { + is := &Items{ + byName: make(map[string]*Item), + byNameAlt: make(map[string]*Item), + items: items, + } + // Fill up byName + for i := range items { + is.byName[Normalize(items[i].Path)] = &items[i] + } + return is +} + +// Find checks off an item +func (is *Items) Find(t *testing.T, obj fs.Object, precision time.Duration) { + remote := Normalize(obj.Remote()) + i, ok := is.byName[remote] + if !ok { + i, ok = is.byNameAlt[remote] + assert.True(t, ok, fmt.Sprintf("Unexpected file %q", remote)) + } + if i != nil { + delete(is.byName, i.Path) + i.Check(t, obj, precision) + } +} + +// Done checks all finished +func (is *Items) Done(t *testing.T) { + if len(is.byName) != 0 { + for name := range is.byName { + t.Logf("Not found %q", name) + } + } + assert.Equal(t, 0, len(is.byName), fmt.Sprintf("%d objects not found", len(is.byName))) +} + +// makeListingFromItems returns a string representation of the items +// +// it returns two possible strings, one normal and one for windows +func makeListingFromItems(items []Item) string { + nameLengths := make([]string, len(items)) + for i, item := range items { + remote := Normalize(item.Path) + nameLengths[i] = fmt.Sprintf("%s (%d)", remote, item.Size) + } + sort.Strings(nameLengths) + return strings.Join(nameLengths, ", ") +} + +// makeListingFromObjects returns a string representation of the objects +func makeListingFromObjects(objs []fs.Object) string { + nameLengths := make([]string, len(objs)) + for i, obj := range objs { + nameLengths[i] = fmt.Sprintf("%s (%d)", Normalize(obj.Remote()), obj.Size()) + } + sort.Strings(nameLengths) + return strings.Join(nameLengths, ", ") +} + +// filterEmptyDirs removes any empty (or containing only directories) +// directories from expectedDirs +func filterEmptyDirs(t *testing.T, items []Item, expectedDirs []string) (newExpectedDirs []string) { + dirs := map[string]struct{}{"": {}} + for _, item := range items { + base := item.Path + for { + base = path.Dir(base) + if base == "." || base == "/" { + break + } + dirs[base] = struct{}{} + } + } + for _, expectedDir := range expectedDirs { + if _, found := dirs[expectedDir]; found { + newExpectedDirs = append(newExpectedDirs, expectedDir) + } else { + t.Logf("Filtering empty directory %q", expectedDir) + } + } + return newExpectedDirs +} + +// CheckListingWithRoot checks the fs to see if it has the +// expected contents with the given precision. +// +// If expectedDirs is non nil then we check those too. Note that no +// directories returned is also OK as some remotes don't return +// directories. +// +// dir is the directory used for the listing. +func CheckListingWithRoot(t *testing.T, f fs.Fs, dir string, items []Item, expectedDirs []string, precision time.Duration) { + if expectedDirs != nil && !f.Features().CanHaveEmptyDirectories { + expectedDirs = filterEmptyDirs(t, items, expectedDirs) + } + is := NewItems(items) + ctx := context.Background() + oldErrors := accounting.Stats(ctx).GetErrors() + var objs []fs.Object + var dirs []fs.Directory + var err error + retries := *ListRetries + sleep := time.Second / 2 + wantListing := makeListingFromItems(items) + gotListing := "<unset>" + listingOK := false + for i := 1; i <= retries; i++ { + objs, dirs, err = walk.GetAll(ctx, f, dir, true, -1) + if err != nil && err != fs.ErrorDirNotFound { + t.Fatalf("Error listing: %v", err) + } + gotListing = makeListingFromObjects(objs) + + listingOK = wantListing == gotListing + if listingOK && (expectedDirs == nil || len(dirs) == len(expectedDirs)) { + // Put an extra sleep in if we did any retries just to make sure it really + // is consistent + if i != 1 { + extraSleep := 5*time.Second + sleep + t.Logf("Sleeping for %v just to make sure", extraSleep) + time.Sleep(extraSleep) + } + break + } + sleep *= 2 + t.Logf("Sleeping for %v for list eventual consistency: %d/%d", sleep, i, retries) + time.Sleep(sleep) + if doDirCacheFlush := f.Features().DirCacheFlush; doDirCacheFlush != nil { + t.Logf("Flushing the directory cache") + doDirCacheFlush() + } + } + assert.True(t, listingOK, fmt.Sprintf("listing wrong, want\n %s got\n %s", wantListing, gotListing)) + for _, obj := range objs { + require.NotNil(t, obj) + is.Find(t, obj, precision) + } + is.Done(t) + // Don't notice an error when listing an empty directory + if len(items) == 0 && oldErrors == 0 && accounting.Stats(ctx).GetErrors() == 1 { + accounting.Stats(ctx).ResetErrors() + } + // Check the directories + if expectedDirs != nil { + expectedDirsCopy := make([]string, len(expectedDirs)) + for i, dir := range expectedDirs { + expectedDirsCopy[i] = Normalize(dir) + } + actualDirs := []string{} + for _, dir := range dirs { + actualDirs = append(actualDirs, Normalize(dir.Remote())) + } + sort.Strings(actualDirs) + sort.Strings(expectedDirsCopy) + assert.Equal(t, expectedDirsCopy, actualDirs, "directories") + } +} + +// CheckListingWithPrecision checks the fs to see if it has the +// expected contents with the given precision. +// +// If expectedDirs is non nil then we check those too. Note that no +// directories returned is also OK as some remotes don't return +// directories. +func CheckListingWithPrecision(t *testing.T, f fs.Fs, items []Item, expectedDirs []string, precision time.Duration) { + CheckListingWithRoot(t, f, "", items, expectedDirs, precision) +} + +// CheckListing checks the fs to see if it has the expected contents +func CheckListing(t *testing.T, f fs.Fs, items []Item) { + precision := f.Precision() + CheckListingWithPrecision(t, f, items, nil, precision) +} + +// CheckItemsWithPrecision checks the fs with the specified precision +// to see if it has the expected items. +func CheckItemsWithPrecision(t *testing.T, f fs.Fs, precision time.Duration, items ...Item) { + CheckListingWithPrecision(t, f, items, nil, precision) +} + +// CheckItems checks the fs to see if it has only the items passed in +// using a precision of fs.Config.ModifyWindow +func CheckItems(t *testing.T, f fs.Fs, items ...Item) { + CheckListingWithPrecision(t, f, items, nil, fs.GetModifyWindow(context.TODO(), f)) +} + +// CompareItems compares a set of DirEntries to a slice of items and a list of dirs +// The modtimes are compared with the precision supplied +func CompareItems(t *testing.T, entries fs.DirEntries, items []Item, expectedDirs []string, precision time.Duration, what string) { + is := NewItems(items) + var objs []fs.Object + var dirs []fs.Directory + wantListing := makeListingFromItems(items) + for _, entry := range entries { + switch x := entry.(type) { + case fs.Directory: + dirs = append(dirs, x) + case fs.Object: + objs = append(objs, x) + // do nothing + default: + t.Fatalf("unknown object type %T", entry) + } + } + + gotListing := makeListingFromObjects(objs) + listingOK := wantListing == gotListing + assert.True(t, listingOK, fmt.Sprintf("%s not equal, want\n %s got\n %s", what, wantListing, gotListing)) + for _, obj := range objs { + require.NotNil(t, obj) + is.Find(t, obj, precision) + } + is.Done(t) + // Check the directories + if expectedDirs != nil { + expectedDirsCopy := make([]string, len(expectedDirs)) + for i, dir := range expectedDirs { + expectedDirsCopy[i] = Normalize(dir) + } + actualDirs := []string{} + for _, dir := range dirs { + actualDirs = append(actualDirs, Normalize(dir.Remote())) + } + sort.Strings(actualDirs) + sort.Strings(expectedDirsCopy) + assert.Equal(t, expectedDirsCopy, actualDirs, "directories not equal") + } +} + +// Time parses a time string or logs a fatal error +func Time(timeString string) time.Time { + t, err := time.Parse(time.RFC3339Nano, timeString) + if err != nil { + fs.Fatalf(nil, "Failed to parse time %q: %v", timeString, err) + } + return t +} + +// LocalRemote creates a temporary directory name for local remotes +func LocalRemote() (path string, err error) { + path, err = os.MkdirTemp("", "rclone") + if err == nil { + // Now remove the directory + err = os.Remove(path) + } + path = filepath.ToSlash(path) + return +} + +// RandomRemoteName makes a random bucket or subdirectory name +// +// Returns a random remote name plus the leaf name +func RandomRemoteName(remoteName string) (string, string, error) { + var err error + var leafName string + + // Make a directory if remote name is null + if remoteName == "" { + remoteName, err = LocalRemote() + if err != nil { + return "", "", err + } + } else { + if !strings.HasSuffix(remoteName, ":") { + remoteName += "/" + } + leafName = "rclone-test-" + random.String(12) + if !MatchTestRemote.MatchString(leafName) { + fs.Fatalf(nil, "%q didn't match the test remote name regexp", leafName) + } + remoteName += leafName + } + return remoteName, leafName, nil +} + +// RandomRemote makes a random bucket or subdirectory on the remote +// from the -remote parameter +// +// Call the finalise function returned to Purge the fs at the end (and +// the parent if necessary) +// +// Returns the remote, its url, a finaliser and an error +func RandomRemote() (fs.Fs, string, func(), error) { + var err error + var parentRemote fs.Fs + remoteName := *RemoteName + + remoteName, _, err = RandomRemoteName(remoteName) + if err != nil { + return nil, "", nil, err + } + + remote, err := fs.NewFs(context.Background(), remoteName) + if err != nil { + return nil, "", nil, err + } + + finalise := func() { + Purge(remote) + if parentRemote != nil { + Purge(parentRemote) + if err != nil { + fs.Logf(nil, "Failed to purge %v: %v", parentRemote, err) + } + } + } + + return remote, remoteName, finalise, nil +} + +// Purge is a simplified re-implementation of operations.Purge for the +// test routine cleanup to avoid circular dependencies. +// +// It logs errors rather than returning them +func Purge(f fs.Fs) { + // Create a stats group here so errors in the cleanup don't + // interfere with the global stats. + ctx := accounting.WithStatsGroup(context.Background(), "test-cleanup") + var err error + doFallbackPurge := true + if doPurge := f.Features().Purge; doPurge != nil { + doFallbackPurge = false + fs.Debugf(f, "Purge remote") + err = doPurge(ctx, "") + if err == fs.ErrorCantPurge { + doFallbackPurge = true + } + } + if doFallbackPurge { + dirs := []string{""} + err = walk.ListR(ctx, f, "", true, -1, walk.ListAll, func(entries fs.DirEntries) error { + var err error + entries.ForObject(func(obj fs.Object) { + fs.Debugf(f, "Purge object %q", obj.Remote()) + err = obj.Remove(ctx) + if err != nil { + fs.Logf(nil, "purge failed to remove %q: %v", obj.Remote(), err) + } + }) + entries.ForDir(func(dir fs.Directory) { + dirs = append(dirs, dir.Remote()) + }) + return nil + }) + sort.Strings(dirs) + for i := len(dirs) - 1; i >= 0; i-- { + dir := dirs[i] + fs.Debugf(f, "Purge dir %q", dir) + err := f.Rmdir(ctx, dir) + if err != nil { + fs.Logf(nil, "purge failed to rmdir %q: %v", dir, err) + } + } + } + if err != nil { + fs.Logf(nil, "purge failed: %v", err) + } +} + +// NewObject finds the object on the remote +func NewObject(ctx context.Context, t *testing.T, f fs.Fs, remote string) fs.Object { + var obj fs.Object + var err error + sleepTime := 1 * time.Second + for i := 1; i <= *ListRetries; i++ { + obj, err = f.NewObject(ctx, remote) + if err == nil { + break + } + t.Logf("Sleeping for %v for findObject eventual consistency: %d/%d (%v)", sleepTime, i, *ListRetries, err) + time.Sleep(sleepTime) + sleepTime = (sleepTime * 3) / 2 + } + require.NoError(t, err) + return obj +} + +// NewDirectoryRetries finds the directory with remote in f +// +// If directory can't be found it returns an error wrapping fs.ErrorDirNotFound +// +// One day this will be an rclone primitive +func NewDirectoryRetries(ctx context.Context, t *testing.T, f fs.Fs, remote string, retries int) (fs.Directory, error) { + var err error + var dir fs.Directory + sleepTime := 1 * time.Second + root := path.Dir(remote) + if root == "." { + root = "" + } + for i := 1; i <= retries; i++ { + var entries fs.DirEntries + entries, err = f.List(ctx, root) + if err != nil { + continue + } + for _, entry := range entries { + var ok bool + dir, ok = entry.(fs.Directory) + if ok && dir.Remote() == remote { + return dir, nil + } + } + err = fmt.Errorf("directory %q not found in %q: %w", remote, root, fs.ErrorDirNotFound) + if i < retries { + t.Logf("Sleeping for %v for NewDirectoryRetries eventual consistency: %d/%d (%v)", sleepTime, i, retries, err) + time.Sleep(sleepTime) + sleepTime = (sleepTime * 3) / 2 + } + } + return dir, err +} + +// NewDirectory finds the directory with remote in f +// +// One day this will be an rclone primitive +func NewDirectory(ctx context.Context, t *testing.T, f fs.Fs, remote string) fs.Directory { + dir, err := NewDirectoryRetries(ctx, t, f, remote, *ListRetries) + require.NoError(t, err) + return dir +} + +// CheckEntryMetadata checks the metadata on the directory +// +// This checks a limited set of metadata on the directory +func CheckEntryMetadata(ctx context.Context, t *testing.T, f fs.Fs, entry fs.DirEntry, wantMeta fs.Metadata) { + features := f.Features() + do, ok := entry.(fs.Metadataer) + require.True(t, ok, "Didn't find expected Metadata() method on %T", entry) + gotMeta, err := do.Metadata(ctx) + require.NoError(t, err) + + for k, v := range wantMeta { + switch k { + case "mtime", "atime", "btime", "ctime": + // Check the system time Metadata + wantT, err := time.Parse(time.RFC3339, v) + require.NoError(t, err) + gotT, err := time.Parse(time.RFC3339, gotMeta[k]) + require.NoError(t, err) + AssertTimeEqualWithPrecision(t, entry.Remote(), wantT, gotT, f.Precision()) + default: + // Check the User metadata if we can + _, isDir := entry.(fs.Directory) + if (isDir && features.UserDirMetadata) || (!isDir && features.UserMetadata) { + assert.Equal(t, v, gotMeta[k]) + } + } + } +} + +// CheckDirModTime checks the modtime on the directory +func CheckDirModTime(ctx context.Context, t *testing.T, f fs.Fs, dir fs.Directory, wantT time.Time) { + if f.Features().DirSetModTime == nil && f.Features().MkdirMetadata == nil { + fs.Debugf(f, "Skipping modtime test as remote does not support DirSetModTime or MkdirMetadata") + return + } + gotT := dir.ModTime(ctx) + precision := f.Precision() + // For unknown reasons the precision of modification times of + // directories on the CI is about >15mS. The tests work fine + // when run in Virtualbox though so I conjecture this is + // something to do with the file system used there. + if runtime.GOOS == "windows" && testy.CI() { + precision = 100 * time.Millisecond + } + AssertTimeEqualWithPrecision(t, dir.Remote(), wantT, gotT, precision) +} + +// Gz returns a compressed version of its input string +func Gz(t *testing.T, s string) string { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + _, err := zw.Write([]byte(s)) + require.NoError(t, err) + err = zw.Close() + require.NoError(t, err) + return buf.String() +} diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go new file mode 100644 index 0000000..4db6f29 --- /dev/null +++ b/fstest/fstests/fstests.go @@ -0,0 +1,2852 @@ +// Package fstests provides generic integration tests for the Fs and +// Object interfaces. +// +// These tests are concerned with the basic functionality of a +// backend. The tests in fs/sync and fs/operations tests more +// cornercases that these tests don't. +package fstests + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "math/bits" + "os" + "path" + "path/filepath" + "reflect" + "slices" + "sort" + "strconv" + "strings" + "testing" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/fspath" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/object" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/fstest/testserver" + "github.com/rclone/rclone/lib/encoder" + "github.com/rclone/rclone/lib/random" + "github.com/rclone/rclone/lib/readers" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// InternalTester is an optional interface for Fs which allows to execute internal tests +// +// This interface should be implemented in 'backend'_internal_test.go and not in 'backend'.go +type InternalTester interface { + InternalTest(*testing.T) +} + +// ChunkedUploadConfig contains the values used by TestFsPutChunked +// to determine the limits of chunked uploading +type ChunkedUploadConfig struct { + // Minimum allowed chunk size + MinChunkSize fs.SizeSuffix + // Maximum allowed chunk size, 0 is no limit + MaxChunkSize fs.SizeSuffix + // Rounds the given chunk size up to the next valid value + // nil will disable rounding + // e.g. the next power of 2 + CeilChunkSize func(fs.SizeSuffix) fs.SizeSuffix + // More than one chunk is required on upload + NeedMultipleChunks bool + // Skip this particular remote + Skip bool +} + +// SetUploadChunkSizer is a test only interface to change the upload chunk size at runtime +type SetUploadChunkSizer interface { + // Change the configured UploadChunkSize. + // Will only be called while no transfer is in progress. + SetUploadChunkSize(fs.SizeSuffix) (fs.SizeSuffix, error) +} + +// SetUploadCutoffer is a test only interface to change the upload cutoff size at runtime +type SetUploadCutoffer interface { + // Change the configured UploadCutoff. + // Will only be called while no transfer is in progress. + SetUploadCutoff(fs.SizeSuffix) (fs.SizeSuffix, error) +} + +// SetCopyCutoffer is a test only interface to change the copy cutoff size at runtime +type SetCopyCutoffer interface { + // Change the configured CopyCutoff. + // Will only be called while no transfer is in progress. + // Return fs.ErrorNotImplemented if you can't implement this + SetCopyCutoff(fs.SizeSuffix) (fs.SizeSuffix, error) +} + +// NextPowerOfTwo returns the current or next bigger power of two. +// All values less or equal 0 will return 0 +func NextPowerOfTwo(i fs.SizeSuffix) fs.SizeSuffix { + return 1 << uint(64-bits.LeadingZeros64(uint64(i)-1)) +} + +// NextMultipleOf returns a function that can be used as a CeilChunkSize function. +// This function will return the next multiple of m that is equal or bigger than i. +// All values less or equal 0 will return 0. +func NextMultipleOf(m fs.SizeSuffix) func(fs.SizeSuffix) fs.SizeSuffix { + if m <= 0 { + panic(fmt.Sprintf("invalid multiplier %s", m)) + } + return func(i fs.SizeSuffix) fs.SizeSuffix { + if i <= 0 { + return 0 + } + + return (((i - 1) / m) + 1) * m + } +} + +// dirsToNames returns a sorted list of names +func dirsToNames(dirs []fs.Directory) []string { + names := []string{} + for _, dir := range dirs { + names = append(names, fstest.Normalize(dir.Remote())) + } + sort.Strings(names) + return names +} + +// objsToNames returns a sorted list of object names +func objsToNames(objs []fs.Object) []string { + names := []string{} + for _, obj := range objs { + names = append(names, fstest.Normalize(obj.Remote())) + } + sort.Strings(names) + return names +} + +// retry f() until no retriable error +func retry(t *testing.T, what string, f func() error) { + const maxTries = 10 + var err error + for tries := 1; tries <= maxTries; tries++ { + err = f() + // exit if no error, or error is not retriable + if err == nil || !fserrors.IsRetryError(err) { + break + } + t.Logf("%s error: %v - low level retry %d/%d", what, err, tries, maxTries) + time.Sleep(2 * time.Second) + } + require.NoError(t, err, what) +} + +// check interface + +// PutTestContentsMetadata puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove +// +// It uploads the object with the mimeType and metadata passed in if set. +// +// It returns the object which will have been checked if check is set +func PutTestContentsMetadata(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, useFileHashes bool, contents string, check bool, mimeType string, metadata fs.Metadata, options ...fs.OpenOption) fs.Object { + var ( + err error + obj fs.Object + uploadHash *hash.MultiHasher + ) + retry(t, "Put", func() error { + buf := bytes.NewBufferString(contents) + uploadHash = hash.NewMultiHasher() + in := io.TeeReader(buf, uploadHash) + + file.Size = int64(buf.Len()) + // The caller explicitly indicates whether the hashes in the file parameter should be used. If hashes is nil, + // then NewStaticObjectInfo will calculate default hashes for use in the check. + hashes := file.Hashes + if !useFileHashes { + hashes = nil + } + obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, hashes, nil) + if mimeType != "" || metadata != nil { + // force the --metadata flag on temporarily + if metadata != nil { + ci := fs.GetConfig(ctx) + previousMetadata := ci.Metadata + ci.Metadata = true + defer func() { + ci.Metadata = previousMetadata + }() + } + obji.WithMetadata(metadata).WithMimeType(mimeType) + } + obj, err = f.Put(ctx, in, obji, options...) + return err + }) + file.Hashes = uploadHash.Sums() + if check { + // Overwrite time with that in metadata if it is already specified + mtime, ok := metadata["mtime"] + if ok { + modTime, err := time.Parse(time.RFC3339Nano, mtime) + require.NoError(t, err) + file.ModTime = modTime + } + file.Check(t, obj, f.Precision()) + // Re-read the object and check again + obj = fstest.NewObject(ctx, t, f, file.Path) + file.Check(t, obj, f.Precision()) + } + return obj +} + +// PutTestContents puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove +func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool) fs.Object { + return PutTestContentsMetadata(ctx, t, f, file, false, contents, check, "", nil) +} + +// testPut puts file with random contents to the remote +func testPut(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) (string, fs.Object) { + return testPutMimeType(ctx, t, f, file, "", nil) +} + +// testPutMimeType puts file with random contents to the remote and the mime type given +func testPutMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, mimeType string, metadata fs.Metadata) (string, fs.Object) { + contents := random.String(100) + // We just generated new contents, but file may contain hashes generated by a previous operation + if len(file.Hashes) > 0 { + file.Hashes = make(map[hash.Type]string) + } + return contents, PutTestContentsMetadata(ctx, t, f, file, false, contents, true, mimeType, metadata) +} + +// testPutLarge puts file to the remote, checks it and removes it on success. +// +// If stream is set, then it uploads the file with size -1 +func testPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, stream bool) { + var ( + err error + obj fs.Object + uploadHash *hash.MultiHasher + ) + retry(t, "PutLarge", func() error { + r := readers.NewPatternReader(file.Size) + uploadHash = hash.NewMultiHasher() + in := io.TeeReader(r, uploadHash) + + size := file.Size + if stream { + size = -1 + } + obji := object.NewStaticObjectInfo(file.Path, file.ModTime, size, true, nil, nil) + obj, err = f.Put(ctx, in, obji) + if file.Size == 0 && err == fs.ErrorCantUploadEmptyFiles { + t.Skip("Can't upload zero length files") + } + return err + }) + file.Hashes = uploadHash.Sums() + file.Check(t, obj, f.Precision()) + + // Re-read the object and check again + obj = fstest.NewObject(ctx, t, f, file.Path) + file.Check(t, obj, f.Precision()) + + // Download the object and check it is OK + downloadHash := hash.NewMultiHasher() + download, err := obj.Open(ctx) + require.NoError(t, err) + n, err := io.Copy(downloadHash, download) + require.NoError(t, err) + assert.Equal(t, file.Size, n) + require.NoError(t, download.Close()) + assert.Equal(t, file.Hashes, downloadHash.Sums()) + + // Remove the object + require.NoError(t, obj.Remove(ctx)) +} + +// TestPutLarge puts file to the remote, checks it and removes it on success. +func TestPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) { + testPutLarge(ctx, t, f, file, false) +} + +// TestPutLargeStreamed puts file of unknown size to the remote, checks it and removes it on success. +func TestPutLargeStreamed(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) { + testPutLarge(ctx, t, f, file, true) +} + +// ReadObject reads the contents of an object as a string +func ReadObject(ctx context.Context, t *testing.T, obj fs.Object, limit int64, options ...fs.OpenOption) string { + what := fmt.Sprintf("readObject(%q) limit=%d, options=%+v", obj, limit, options) + in, err := obj.Open(ctx, options...) + require.NoError(t, err, what) + var r io.Reader = in + if limit >= 0 { + r = &io.LimitedReader{R: r, N: limit} + } + contents, err := io.ReadAll(r) + require.NoError(t, err, what) + err = in.Close() + require.NoError(t, err, what) + return string(contents) +} + +// ExtraConfigItem describes a config item for the tests +type ExtraConfigItem struct{ Name, Key, Value string } + +// Opt is options for Run +type Opt struct { + RemoteName string + NilObject fs.Object + ExtraConfig []ExtraConfigItem + SkipBadWindowsCharacters bool // skips unusable characters for windows if set + SkipFsMatch bool // if set skip exact matching of Fs value + TiersToTest []string // List of tiers which can be tested in setTier test + ChunkedUpload ChunkedUploadConfig + UnimplementableFsMethods []string // List of Fs methods which can't be implemented in this wrapping Fs + UnimplementableObjectMethods []string // List of Object methods which can't be implemented in this wrapping Fs + UnimplementableDirectoryMethods []string // List of Directory methods which can't be implemented in this wrapping Fs + SkipFsCheckWrap bool // if set skip FsCheckWrap + SkipObjectCheckWrap bool // if set skip ObjectCheckWrap + SkipDirectoryCheckWrap bool // if set skip DirectoryCheckWrap + SkipInvalidUTF8 bool // if set skip invalid UTF-8 checks + SkipLeadingDot bool // if set skip leading dot checks + QuickTestOK bool // if set, run this test with make quicktest +} + +// returns true if x is found in ss +func stringsContains(x string, ss []string) bool { + return slices.Contains(ss, x) +} + +// toUpperASCII returns a copy of the string s with all Unicode +// letters mapped to their upper case. +func toUpperASCII(s string) string { + return strings.Map(func(r rune) rune { + if 'a' <= r && r <= 'z' { + r -= 'a' - 'A' + } + return r + }, s) +} + +// removeConfigID removes any {xyz} parts of the name put in for +// config disambiguation +func removeConfigID(s string) string { + bra := strings.IndexRune(s, '{') + ket := strings.IndexRune(s, '}') + if bra >= 0 && ket > bra { + s = s[:bra] + s[ket+1:] + } + return s +} + +// InternalTestFiles is the state of the remote at the moment the internal tests are called +var InternalTestFiles []fstest.Item + +// Run runs the basic integration tests for a remote using the options passed in. +// +// They are structured in a hierarchical way so that dependencies for the tests can be created. +// +// For example some tests require the directory to be created - these +// are inside the "FsMkdir" test. Some tests require some tests files +// - these are inside the "FsPutFiles" test. +func Run(t *testing.T, opt *Opt) { + var ( + f fs.Fs + remoteName = opt.RemoteName + subRemoteName string + subRemoteLeaf string + file1 = fstest.Item{ + ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), + Path: "file name.txt", + } + file1Contents string + file1MimeType = "text/csv" + file1Metadata = fs.Metadata{"rclonetest": "potato"} + file2 = fstest.Item{ + ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"), + Path: `hello? sausage/ĂªĂ©/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`, + } + isLocalRemote bool + purged bool // whether the dir has been purged or not + ctx = context.Background() + ci = fs.GetConfig(ctx) + unwrappableFsMethods = []string{"Command"} // these Fs methods don't need to be wrapped ever + ) + + if strings.HasSuffix(os.Getenv("RCLONE_CONFIG"), "/notfound") && *fstest.RemoteName == "" && !opt.QuickTestOK { + t.Skip("quicktest only") + } + + // Skip the test if the remote isn't configured + skipIfNotOk := func(t *testing.T) { + if f == nil { + t.Skipf("WARN: %q not configured", remoteName) + } + } + + // Skip if remote is not ListR capable, otherwise set the useListR + // flag, returning a function to restore its value + skipIfNotListR := func(t *testing.T) func() { + skipIfNotOk(t) + if f.Features().ListR == nil { + t.Skip("FS has no ListR interface") + } + previous := ci.UseListR + ci.UseListR = true + return func() { + ci.UseListR = previous + } + } + + // Skip if remote is not SetTier and GetTier capable + skipIfNotSetTier := func(t *testing.T) { + skipIfNotOk(t) + if !f.Features().SetTier || !f.Features().GetTier { + t.Skip("FS has no SetTier & GetTier interfaces") + } + } + + // Return true if f (or any of the things it wraps) is bucket + // based but not at the root. + isBucketBasedButNotRoot := func(f fs.Fs) bool { + f = fs.UnWrapFs(f) + return f.Features().BucketBased && strings.Contains(strings.Trim(f.Root(), "/"), "/") + } + + // Initialise the remote + fstest.Initialise() + + // Set extra config if supplied + for _, item := range opt.ExtraConfig { + config.FileSetValue(item.Name, item.Key, item.Value) + } + if *fstest.RemoteName != "" { + remoteName = *fstest.RemoteName + } + oldFstestRemoteName := fstest.RemoteName + fstest.RemoteName = &remoteName + defer func() { + fstest.RemoteName = oldFstestRemoteName + }() + t.Logf("Using remote %q", remoteName) + var err error + if remoteName == "" { + remoteName, err = fstest.LocalRemote() + require.NoError(t, err) + isLocalRemote = true + } + + // Start any test servers if required + finish, err := testserver.Start(remoteName) + require.NoError(t, err) + defer finish() + + // Make the Fs we are testing with, initialising the local variables + // subRemoteName - name of the remote after the TestRemote: + // subRemoteLeaf - a subdirectory to use under that + // remote - the result of fs.NewFs(TestRemote:subRemoteName) + subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(remoteName) + require.NoError(t, err) + f, err = fs.NewFs(context.Background(), subRemoteName) + if errors.Is(err, fs.ErrorNotFoundInConfigFile) { + t.Logf("Didn't find %q in config file - skipping tests", remoteName) + return + } + require.NoError(t, err, fmt.Sprintf("unexpected error: %v", err)) + + // Get fsInfo which contains type, etc. of the fs + fsInfo, _, _, _, err := fs.ConfigFs(subRemoteName) + require.NoError(t, err, fmt.Sprintf("unexpected error: %v", err)) + + // Skip the rest if it failed + skipIfNotOk(t) + + // Check to see if Fs that wrap other Fs implement all the optional methods + t.Run("FsCheckWrap", func(t *testing.T) { + skipIfNotOk(t) + if opt.SkipFsCheckWrap { + t.Skip("Skipping FsCheckWrap on this Fs") + } + ft := new(fs.Features).Fill(ctx, f) + if ft.UnWrap == nil && !f.Features().Overlay { + t.Skip("Not a wrapping Fs") + } + v := reflect.ValueOf(ft).Elem() + vType := v.Type() + for i := range v.NumField() { + vName := vType.Field(i).Name + if stringsContains(vName, opt.UnimplementableFsMethods) { + continue + } + if stringsContains(vName, unwrappableFsMethods) { + continue + } + field := v.Field(i) + // skip the bools + if field.Type().Kind() == reflect.Bool { + continue + } + if field.IsNil() { + t.Errorf("Missing Fs wrapper for %s", vName) + } + } + }) + + // Check to see if Fs advertises commands and they work and have docs + t.Run("FsCommand", func(t *testing.T) { + skipIfNotOk(t) + doCommand := f.Features().Command + if doCommand == nil { + t.Skip("No commands in this remote") + } + // Check the correct error is generated + _, err := doCommand(context.Background(), "NOTFOUND", nil, nil) + assert.Equal(t, fs.ErrorCommandNotFound, err, "Incorrect error generated on command not found") + // Check there are some commands in the fsInfo + fsInfo, _, _, _, err := fs.ConfigFs(remoteName) + require.NoError(t, err) + assert.True(t, len(fsInfo.CommandHelp) > 0, "Command is declared, must return some help in CommandHelp") + }) + + // TestFsRmdirNotFound tests deleting a nonexistent directory + t.Run("FsRmdirNotFound", func(t *testing.T) { + skipIfNotOk(t) + if isBucketBasedButNotRoot(f) { + t.Skip("Skipping test as non root bucket-based remote") + } + err := f.Rmdir(ctx, "") + assert.Error(t, err, "Expecting error on Rmdir nonexistent") + }) + + // Make the directory + err = f.Mkdir(ctx, "") + require.NoError(t, err) + fstest.CheckListing(t, f, []fstest.Item{}) + + // TestFsString tests the String method + t.Run("FsString", func(t *testing.T) { + skipIfNotOk(t) + str := f.String() + require.NotEqual(t, "", str) + }) + + // TestFsName tests the Name method + t.Run("FsName", func(t *testing.T) { + skipIfNotOk(t) + got := removeConfigID(f.Name()) + var want string + if isLocalRemote { + want = "local" + } else { + want = remoteName[:strings.LastIndex(remoteName, ":")] + comma := strings.IndexRune(remoteName, ',') + if comma >= 0 { + want = want[:comma] + } + } + require.Equal(t, want, got) + }) + + // TestFsRoot tests the Root method + t.Run("FsRoot", func(t *testing.T) { + skipIfNotOk(t) + got := f.Root() + want := subRemoteName + colon := strings.LastIndex(want, ":") + if colon >= 0 { + want = want[colon+1:] + } + if isLocalRemote { + // only check last path element on local + require.Equal(t, filepath.Base(subRemoteName), filepath.Base(got)) + } else { + require.Equal(t, want, got) + } + }) + + // TestFsRmdirEmpty tests deleting an empty directory + t.Run("FsRmdirEmpty", func(t *testing.T) { + skipIfNotOk(t) + err := f.Rmdir(ctx, "") + require.NoError(t, err) + }) + + // TestFsMkdir tests making a directory + // + // Tests that require the directory to be made are within this + t.Run("FsMkdir", func(t *testing.T) { + skipIfNotOk(t) + + err := f.Mkdir(ctx, "") + require.NoError(t, err) + fstest.CheckListing(t, f, []fstest.Item{}) + + err = f.Mkdir(ctx, "") + require.NoError(t, err) + + // TestFsMkdirRmdirSubdir tests making and removing a sub directory + t.Run("FsMkdirRmdirSubdir", func(t *testing.T) { + skipIfNotOk(t) + dir := "dir/subdir" + err := operations.Mkdir(ctx, f, dir) + require.NoError(t, err) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{"dir", "dir/subdir"}, fs.GetModifyWindow(ctx, f)) + + err = operations.Rmdir(ctx, f, dir) + require.NoError(t, err) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{"dir"}, fs.GetModifyWindow(ctx, f)) + + err = operations.Rmdir(ctx, f, "dir") + require.NoError(t, err) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f)) + }) + + // TestFsListEmpty tests listing an empty directory + t.Run("FsListEmpty", func(t *testing.T) { + skipIfNotOk(t) + fstest.CheckListing(t, f, []fstest.Item{}) + }) + + // TestFsListDirEmpty tests listing the directories from an empty directory + TestFsListDirEmpty := func(t *testing.T) { + skipIfNotOk(t) + objs, dirs, err := walk.GetAll(ctx, f, "", true, 1) + if !f.Features().CanHaveEmptyDirectories { + if err != fs.ErrorDirNotFound { + require.NoError(t, err) + } + } else { + require.NoError(t, err) + } + assert.Equal(t, []string{}, objsToNames(objs)) + assert.Equal(t, []string{}, dirsToNames(dirs)) + } + t.Run("FsListDirEmpty", TestFsListDirEmpty) + + // TestFsListRDirEmpty tests listing the directories from an empty directory using ListR + t.Run("FsListRDirEmpty", func(t *testing.T) { + defer skipIfNotListR(t)() + TestFsListDirEmpty(t) + }) + + // TestFsListDirNotFound tests listing the directories from an empty directory + TestFsListDirNotFound := func(t *testing.T) { + skipIfNotOk(t) + objs, dirs, err := walk.GetAll(ctx, f, "does not exist", true, 1) + if !f.Features().CanHaveEmptyDirectories { + if err != fs.ErrorDirNotFound { + assert.NoError(t, err) + assert.Equal(t, 0, len(objs)+len(dirs)) + } + } else { + assert.Equal(t, fs.ErrorDirNotFound, err) + } + } + t.Run("FsListDirNotFound", TestFsListDirNotFound) + + // TestFsListRDirNotFound tests listing the directories from an empty directory using ListR + t.Run("FsListRDirNotFound", func(t *testing.T) { + defer skipIfNotListR(t)() + TestFsListDirNotFound(t) + }) + + // FsEncoding tests that file name encodings are + // working by uploading a series of unusual files + // Must be run in an empty directory + t.Run("FsEncoding", func(t *testing.T) { + skipIfNotOk(t) + if testing.Short() { + t.Skip("not running with -short") + } + + // check no files or dirs as pre-requisite + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f)) + + for _, test := range []struct { + name string + path string + }{ + // See lib/encoder/encoder.go for list of things that go here + {"control chars", "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\x7F"}, + {"dot", "."}, + {"dot dot", ".."}, + {"punctuation", "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"}, + {"leading space", " leading space"}, + {"leading tilde", "~leading tilde"}, + {"leading CR", "\rleading CR"}, + {"leading LF", "\nleading LF"}, + {"leading HT", "\tleading HT"}, + {"leading VT", "\vleading VT"}, + {"leading dot", ".leading dot"}, + {"trailing space", "trailing space "}, + {"trailing CR", "trailing CR\r"}, + {"trailing LF", "trailing LF\n"}, + {"trailing HT", "trailing HT\t"}, + {"trailing VT", "trailing VT\v"}, + {"trailing dot", "trailing dot."}, + {"invalid UTF-8", "invalid utf-8\xfe"}, + {"URL encoding", "test%46.txt"}, + } { + t.Run(test.name, func(t *testing.T) { + if opt.SkipInvalidUTF8 && test.name == "invalid UTF-8" { + t.Skip("Skipping " + test.name) + } + if opt.SkipLeadingDot && test.name == "leading dot" { + t.Skip("Skipping " + test.name) + } + + // turn raw strings into Standard encoding + fileName := encoder.Standard.Encode(test.path) + dirName := fileName + t.Logf("testing %q", fileName) + assert.NoError(t, f.Mkdir(ctx, dirName)) + file := fstest.Item{ + ModTime: time.Now(), + Path: dirName + "/" + fileName, // test creating a file and dir with that name + } + _, o := testPut(context.Background(), t, f, &file) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file}, []string{dirName}, fs.GetModifyWindow(ctx, f)) + assert.NoError(t, o.Remove(ctx)) + assert.NoError(t, f.Rmdir(ctx, dirName)) + fstest.CheckListingWithPrecision(t, f, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, f)) + }) + } + }) + + // TestFsNewObjectNotFound tests not finding an object + t.Run("FsNewObjectNotFound", func(t *testing.T) { + skipIfNotOk(t) + // Object in an existing directory + o, err := f.NewObject(ctx, "potato") + assert.Nil(t, o) + assert.Equal(t, fs.ErrorObjectNotFound, err) + // Now try an object in a nonexistent directory + o, err = f.NewObject(ctx, "directory/not/found/potato") + assert.Nil(t, o) + assert.Equal(t, fs.ErrorObjectNotFound, err) + }) + + // TestFsPutError tests uploading a file where there is an error + // + // It makes sure that aborting a file half way through does not create + // a file on the remote. + // + // go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutError)$' + t.Run("FsPutError", func(t *testing.T) { + skipIfNotOk(t) + + var N int64 = 5 * 1024 + if *fstest.SizeLimit > 0 && N > *fstest.SizeLimit { + N = *fstest.SizeLimit + t.Logf("Reduce file size due to limit %d", N) + } + + // Read N bytes then produce an error + contents := random.String(int(N)) + buf := bytes.NewBufferString(contents) + er := &readers.ErrorReader{Err: errors.New("potato")} + in := io.MultiReader(buf, er) + + obji := object.NewStaticObjectInfo(file2.Path, file2.ModTime, 2*N, true, nil, nil) + _, err := f.Put(ctx, in, obji) + // assert.Nil(t, obj) - FIXME some remotes return the object even on nil + assert.NotNil(t, err) + + retry(t, "FsPutError: test object does not exist", func() error { + obj, err := f.NewObject(ctx, file2.Path) + if err == nil { + return fserrors.RetryErrorf("object is present") + } + assert.Nil(t, obj) + assert.Equal(t, fs.ErrorObjectNotFound, err) + return nil + }) + }) + + t.Run("FsPutZeroLength", func(t *testing.T) { + skipIfNotOk(t) + + TestPutLarge(ctx, t, f, &fstest.Item{ + ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), + Path: "zero-length-file", + Size: int64(0), + }) + }) + + t.Run("FsOpenWriterAt", func(t *testing.T) { + skipIfNotOk(t) + openWriterAt := f.Features().OpenWriterAt + if openWriterAt == nil { + t.Skip("FS has no OpenWriterAt interface") + } + path := "writer-at-subdir/writer-at-file" + out, err := openWriterAt(ctx, path, -1) + require.NoError(t, err) + + var n int + n, err = out.WriteAt([]byte("def"), 3) + assert.NoError(t, err) + assert.Equal(t, 3, n) + n, err = out.WriteAt([]byte("ghi"), 6) + assert.NoError(t, err) + assert.Equal(t, 3, n) + n, err = out.WriteAt([]byte("abc"), 0) + assert.NoError(t, err) + assert.Equal(t, 3, n) + + assert.NoError(t, out.Close()) + + obj := fstest.NewObject(ctx, t, f, path) + assert.Equal(t, "abcdefghi", ReadObject(ctx, t, obj, -1), "contents of file differ") + + assert.NoError(t, obj.Remove(ctx)) + assert.NoError(t, f.Rmdir(ctx, "writer-at-subdir")) + }) + + // TestFsOpenChunkWriter tests writing in chunks to fs + // then reads back the contents and check if they match + // go test -v -run 'TestIntegration/FsMkdir/FsOpenChunkWriter' + t.Run("FsOpenChunkWriter", func(t *testing.T) { + skipIfNotOk(t) + openChunkWriter := f.Features().OpenChunkWriter + if openChunkWriter == nil { + t.Skip("FS has no OpenChunkWriter interface") + } + size5MBs := 5 * 1024 * 1024 + contents1 := random.String(size5MBs) + contents2 := random.String(size5MBs) + + size1MB := 1 * 1024 * 1024 + contents3 := random.String(size1MB) + + path := "writer-at-subdir/writer-at-file" + objSrc := object.NewStaticObjectInfo(path+"-WRONG-REMOTE", file1.ModTime, -1, true, nil, nil) + _, out, err := openChunkWriter(ctx, path, objSrc, &fs.ChunkOption{ + ChunkSize: int64(size5MBs), + }) + require.NoError(t, err) + + var n int64 + n, err = out.WriteChunk(ctx, 1, strings.NewReader(contents2)) + assert.NoError(t, err) + assert.Equal(t, int64(size5MBs), n) + n, err = out.WriteChunk(ctx, 2, strings.NewReader(contents3)) + assert.NoError(t, err) + assert.Equal(t, int64(size1MB), n) + n, err = out.WriteChunk(ctx, 0, strings.NewReader(contents1)) + assert.NoError(t, err) + assert.Equal(t, int64(size5MBs), n) + + assert.NoError(t, out.Close(ctx)) + + obj := fstest.NewObject(ctx, t, f, path) + originalContents := contents1 + contents2 + contents3 + fileContents := ReadObject(ctx, t, obj, -1) + isEqual := originalContents == fileContents + assert.True(t, isEqual, "contents of file differ") + + assert.NoError(t, obj.Remove(ctx)) + assert.NoError(t, f.Rmdir(ctx, "writer-at-subdir")) + }) + + // TestFsChangeNotify tests that changes are properly + // propagated + // + // go test -v -remote TestDrive: -run '^Test(Setup|Init|FsChangeNotify)$' -verbose + t.Run("FsChangeNotify", func(t *testing.T) { + skipIfNotOk(t) + + // Check have ChangeNotify + doChangeNotify := f.Features().ChangeNotify + if doChangeNotify == nil { + t.Skip("FS has no ChangeNotify interface") + } + + err := operations.Mkdir(ctx, f, "dir") + require.NoError(t, err) + + pollInterval := make(chan time.Duration) + dirChanges := map[string]struct{}{} + objChanges := map[string]struct{}{} + doChangeNotify(ctx, func(x string, e fs.EntryType) { + fs.Debugf(nil, "doChangeNotify(%q, %+v)", x, e) + if strings.HasPrefix(x, file1.Path[:5]) || strings.HasPrefix(x, file2.Path[:5]) { + fs.Debugf(nil, "Ignoring notify for file1 or file2: %q, %v", x, e) + return + } + if e == fs.EntryDirectory { + dirChanges[x] = struct{}{} + } else if e == fs.EntryObject { + objChanges[x] = struct{}{} + } + }, pollInterval) + defer func() { close(pollInterval) }() + pollInterval <- time.Second + + var dirs []string + for _, idx := range []int{1, 3, 2} { + dir := fmt.Sprintf("dir/subdir%d", idx) + err = operations.Mkdir(ctx, f, dir) + require.NoError(t, err) + dirs = append(dirs, dir) + } + + var objs []fs.Object + for _, idx := range []int{2, 4, 3} { + file := fstest.Item{ + ModTime: time.Now(), + Path: fmt.Sprintf("dir/file%d", idx), + } + _, o := testPut(ctx, t, f, &file) + objs = append(objs, o) + } + + // Looks for each item in wants in changes - + // if they are all found it returns true + contains := func(changes map[string]struct{}, wants []string) bool { + for _, want := range wants { + _, ok := changes[want] + if !ok { + return false + } + } + return true + } + + // Wait a little while for the changes to come in + wantDirChanges := []string{"dir/subdir1", "dir/subdir3", "dir/subdir2"} + wantObjChanges := []string{"dir/file2", "dir/file4", "dir/file3"} + ok := false + for tries := 1; tries < 10; tries++ { + ok = contains(dirChanges, wantDirChanges) && contains(objChanges, wantObjChanges) + if ok { + break + } + t.Logf("Try %d/10 waiting for dirChanges and objChanges", tries) + time.Sleep(3 * time.Second) + } + if !ok { + t.Errorf("%+v does not contain %+v or \n%+v does not contain %+v", dirChanges, wantDirChanges, objChanges, wantObjChanges) + } + + // tidy up afterwards + for _, o := range objs { + assert.NoError(t, o.Remove(ctx)) + } + dirs = append(dirs, "dir") + for _, dir := range dirs { + assert.NoError(t, f.Rmdir(ctx, dir)) + } + }) + + // TestFsPut files writes file1, file2 and tests an update + // + // Tests that require file1, file2 are within this + t.Run("FsPutFiles", func(t *testing.T) { + skipIfNotOk(t) + file1Contents, _ = testPut(ctx, t, f, &file1) + /* file2Contents = */ testPut(ctx, t, f, &file2) + file1Contents, _ = testPutMimeType(ctx, t, f, &file1, file1MimeType, file1Metadata) + // Note that the next test will check there are no duplicated file names + + // TestFsListDirFile2 tests the files are correctly uploaded by doing + // Depth 1 directory listings + TestFsListDirFile2 := func(t *testing.T) { + skipIfNotOk(t) + list := func(dir string, expectedDirNames, expectedObjNames []string) { + var objNames, dirNames []string + for i := 1; i <= *fstest.ListRetries; i++ { + objs, dirs, err := walk.GetAll(ctx, f, dir, true, 1) + if errors.Is(err, fs.ErrorDirNotFound) { + objs, dirs, err = walk.GetAll(ctx, f, dir, true, 1) + } + require.NoError(t, err) + objNames = objsToNames(objs) + dirNames = dirsToNames(dirs) + if len(objNames) >= len(expectedObjNames) && len(dirNames) >= len(expectedDirNames) { + break + } + t.Logf("Sleeping for 1 second for TestFsListDirFile2 eventual consistency: %d/%d", i, *fstest.ListRetries) + time.Sleep(1 * time.Second) + } + assert.Equal(t, expectedDirNames, dirNames) + assert.Equal(t, expectedObjNames, objNames) + } + dir := file2.Path + deepest := true + for dir != "" { + expectedObjNames := []string{} + expectedDirNames := []string{} + child := dir + dir = path.Dir(dir) + if dir == "." { + dir = "" + expectedObjNames = append(expectedObjNames, file1.Path) + } + if deepest { + expectedObjNames = append(expectedObjNames, file2.Path) + deepest = false + } else { + expectedDirNames = append(expectedDirNames, child) + } + list(dir, expectedDirNames, expectedObjNames) + } + } + t.Run("FsListDirFile2", TestFsListDirFile2) + + // TestFsListRDirFile2 tests the files are correctly uploaded by doing + // Depth 1 directory listings using ListR + t.Run("FsListRDirFile2", func(t *testing.T) { + defer skipIfNotListR(t)() + TestFsListDirFile2(t) + }) + + // Test the files are all there with walk.ListR recursive listings + t.Run("FsListR", func(t *testing.T) { + skipIfNotOk(t) + objs, dirs, err := walk.GetAll(ctx, f, "", true, -1) + require.NoError(t, err) + assert.Equal(t, []string{ + "hello? sausage", + "hello? sausage/ĂªĂ©", + "hello? sausage/ĂªĂ©/Hello, 世界", + "hello? sausage/ĂªĂ©/Hello, 世界/ \" ' @ < > & ? + ≠", + }, dirsToNames(dirs)) + assert.Equal(t, []string{ + "file name.txt", + "hello? sausage/ĂªĂ©/Hello, 世界/ \" ' @ < > & ? + ≠/z.txt", + }, objsToNames(objs)) + }) + + // Test the files are all there with + // walk.ListR recursive listings on a sub dir + t.Run("FsListRSubdir", func(t *testing.T) { + skipIfNotOk(t) + objs, dirs, err := walk.GetAll(ctx, f, path.Dir(path.Dir(path.Dir(path.Dir(file2.Path)))), true, -1) + require.NoError(t, err) + assert.Equal(t, []string{ + "hello? sausage/ĂªĂ©", + "hello? sausage/ĂªĂ©/Hello, 世界", + "hello? sausage/ĂªĂ©/Hello, 世界/ \" ' @ < > & ? + ≠", + }, dirsToNames(dirs)) + assert.Equal(t, []string{ + "hello? sausage/ĂªĂ©/Hello, 世界/ \" ' @ < > & ? + ≠/z.txt", + }, objsToNames(objs)) + }) + + // TestFsListDirRoot tests that DirList works in the root + TestFsListDirRoot := func(t *testing.T) { + skipIfNotOk(t) + rootRemote, err := fs.NewFs(context.Background(), remoteName) + require.NoError(t, err) + _, dirs, err := walk.GetAll(ctx, rootRemote, "", true, 1) + require.NoError(t, err) + assert.Contains(t, dirsToNames(dirs), subRemoteLeaf, "Remote leaf not found") + } + t.Run("FsListDirRoot", TestFsListDirRoot) + + // TestFsListRDirRoot tests that DirList works in the root using ListR + t.Run("FsListRDirRoot", func(t *testing.T) { + defer skipIfNotListR(t)() + TestFsListDirRoot(t) + }) + + // TestFsListSubdir tests List works for a subdirectory + TestFsListSubdir := func(t *testing.T) { + skipIfNotOk(t) + fileName := file2.Path + var err error + var objs []fs.Object + var dirs []fs.Directory + for range 2 { + dir, _ := path.Split(fileName) + dir = dir[:len(dir)-1] + objs, dirs, err = walk.GetAll(ctx, f, dir, true, -1) + } + require.NoError(t, err) + require.Len(t, objs, 1) + assert.Equal(t, fileName, objs[0].Remote()) + require.Len(t, dirs, 0) + } + t.Run("FsListSubdir", TestFsListSubdir) + + // TestFsListRSubdir tests List works for a subdirectory using ListR + t.Run("FsListRSubdir", func(t *testing.T) { + defer skipIfNotListR(t)() + TestFsListSubdir(t) + }) + + // TestFsListLevel2 tests List works for 2 levels + TestFsListLevel2 := func(t *testing.T) { + skipIfNotOk(t) + objs, dirs, err := walk.GetAll(ctx, f, "", true, 2) + if err == fs.ErrorLevelNotSupported { + return + } + require.NoError(t, err) + assert.Equal(t, []string{file1.Path}, objsToNames(objs)) + assert.Equal(t, []string{"hello? sausage", "hello? sausage/ĂªĂ©"}, dirsToNames(dirs)) + } + t.Run("FsListLevel2", TestFsListLevel2) + + // TestFsListRLevel2 tests List works for 2 levels using ListR + t.Run("FsListRLevel2", func(t *testing.T) { + defer skipIfNotListR(t)() + TestFsListLevel2(t) + }) + + // TestFsListFile1 tests file present + t.Run("FsListFile1", func(t *testing.T) { + skipIfNotOk(t) + fstest.CheckListing(t, f, []fstest.Item{file1, file2}) + }) + + // TestFsNewObject tests NewObject + t.Run("FsNewObject", func(t *testing.T) { + skipIfNotOk(t) + obj := fstest.NewObject(ctx, t, f, file1.Path) + file1.Check(t, obj, f.Precision()) + }) + + // FsNewObjectCaseInsensitive tests NewObject on a case insensitive file system + t.Run("FsNewObjectCaseInsensitive", func(t *testing.T) { + skipIfNotOk(t) + if !f.Features().CaseInsensitive { + t.Skip("Not Case Insensitive") + } + obj := fstest.NewObject(ctx, t, f, toUpperASCII(file1.Path)) + file1.Check(t, obj, f.Precision()) + t.Run("Dir", func(t *testing.T) { + obj := fstest.NewObject(ctx, t, f, toUpperASCII(file2.Path)) + file2.Check(t, obj, f.Precision()) + }) + }) + + // TestFsListFile1and2 tests two files present + t.Run("FsListFile1and2", func(t *testing.T) { + skipIfNotOk(t) + fstest.CheckListing(t, f, []fstest.Item{file1, file2}) + }) + + // TestFsNewObjectDir tests NewObject on a directory which should produce fs.ErrorIsDir if possible or fs.ErrorObjectNotFound if not + t.Run("FsNewObjectDir", func(t *testing.T) { + skipIfNotOk(t) + dir := path.Dir(file2.Path) + obj, err := f.NewObject(ctx, dir) + assert.Nil(t, obj) + assert.True(t, err == fs.ErrorIsDir || err == fs.ErrorObjectNotFound, fmt.Sprintf("Wrong error: expecting fs.ErrorIsDir or fs.ErrorObjectNotFound but got: %#v", err)) + }) + + // TestFsPurge tests Purge + t.Run("FsPurge", func(t *testing.T) { + skipIfNotOk(t) + + // Check have Purge + doPurge := f.Features().Purge + if doPurge == nil { + t.Skip("FS has no Purge interface") + } + + // put up a file to purge + fileToPurge := fstest.Item{ + ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), + Path: "dirToPurge/fileToPurge.txt", + } + _, _ = testPut(ctx, t, f, &fileToPurge) + + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2, fileToPurge}, []string{ + "dirToPurge", + "hello? sausage", + "hello? sausage/ĂªĂ©", + "hello? sausage/ĂªĂ©/Hello, 世界", + "hello? sausage/ĂªĂ©/Hello, 世界/ \" ' @ < > & ? + ≠", + }, fs.GetModifyWindow(ctx, f)) + + // Now purge it + err = operations.Purge(ctx, f, "dirToPurge") + require.NoError(t, err) + + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2}, []string{ + "hello? sausage", + "hello? sausage/ĂªĂ©", + "hello? sausage/ĂªĂ©/Hello, 世界", + "hello? sausage/ĂªĂ©/Hello, 世界/ \" ' @ < > & ? + ≠", + }, fs.GetModifyWindow(ctx, f)) + }) + + // TestFsPurge tests Purge on the Root + t.Run("FsPurgeRoot", func(t *testing.T) { + skipIfNotOk(t) + + // Check have Purge + doPurge := f.Features().Purge + if doPurge == nil { + t.Skip("FS has no Purge interface") + } + + // put up a file to purge + fileToPurge := fstest.Item{ + ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), + Path: "dirToPurgeFromRoot/fileToPurgeFromRoot.txt", + } + _, _ = testPut(ctx, t, f, &fileToPurge) + + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2, fileToPurge}, []string{ + "dirToPurgeFromRoot", + "hello? sausage", + "hello? sausage/ĂªĂ©", + "hello? sausage/ĂªĂ©/Hello, 世界", + "hello? sausage/ĂªĂ©/Hello, 世界/ \" ' @ < > & ? + ≠", + }, fs.GetModifyWindow(ctx, f)) + + // Create a new Fs pointing at the directory + remoteName := subRemoteName + "/" + "dirToPurgeFromRoot" + fPurge, err := fs.NewFs(context.Background(), remoteName) + require.NoError(t, err) + + // Now purge it from the root + err = operations.Purge(ctx, fPurge, "") + require.NoError(t, err) + + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2}, []string{ + "hello? sausage", + "hello? sausage/ĂªĂ©", + "hello? sausage/ĂªĂ©/Hello, 世界", + "hello? sausage/ĂªĂ©/Hello, 世界/ \" ' @ < > & ? + ≠", + }, fs.GetModifyWindow(ctx, f)) + }) + + // TestFsListRootedSubdir tests putting and listing with an Fs that is rooted at a subdirectory 2 levels down + TestFsListRootedSubdir := func(t *testing.T) { + skipIfNotOk(t) + newF, err := cache.Get(ctx, subRemoteName+"/hello? sausage/ĂªĂ©") + assert.NoError(t, err) + nestedFile := fstest.Item{ + ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), + Path: "a/b/c/d/e.txt", + } + _, _ = testPut(ctx, t, newF, &nestedFile) + + objs, dirs, err := walk.GetAll(ctx, newF, "", true, 10) + require.NoError(t, err) + assert.Equal(t, []string{`Hello, 世界/ " ' @ < > & ? + ≠/z.txt`, nestedFile.Path}, objsToNames(objs)) + assert.Equal(t, []string{`Hello, 世界`, `Hello, 世界/ " ' @ < > & ? + ≠`, "a", "a/b", "a/b/c", "a/b/c/d"}, dirsToNames(dirs)) + + // cleanup + err = operations.Purge(ctx, newF, "a") + require.NoError(t, err) + } + t.Run("FsListRootedSubdir", TestFsListRootedSubdir) + + // TestFsCopy tests Copy + t.Run("FsCopy", func(t *testing.T) { + skipIfNotOk(t) + + // Check have Copy + doCopy := f.Features().Copy + if doCopy == nil { + t.Skip("FS has no Copier interface") + } + + // Test with file2 so have + and ' ' in file name + var file2Copy = file2 + file2Copy.Path += "-copy" + + // do the copy + src := fstest.NewObject(ctx, t, f, file2.Path) + dst, err := doCopy(ctx, src, file2Copy.Path) + if err == fs.ErrorCantCopy { + t.Skip("FS can't copy") + } + require.NoError(t, err, fmt.Sprintf("Error: %#v", err)) + + // check file exists in new listing + fstest.CheckListing(t, f, []fstest.Item{file1, file2, file2Copy}) + + // Check dst lightly - list above has checked ModTime/Hashes + assert.Equal(t, file2Copy.Path, dst.Remote()) + + // check that mutating dst does not mutate src + if !strings.Contains(fs.ConfigStringFull(f), "copy_is_hardlink") { + err = dst.SetModTime(ctx, fstest.Time("2004-03-03T04:05:06.499999999Z")) + if err != fs.ErrorCantSetModTimeWithoutDelete && err != fs.ErrorCantSetModTime { + assert.NoError(t, err) + // Re-read the source and check its modtime + src = fstest.NewObject(ctx, t, f, src.Remote()) + assert.False(t, src.ModTime(ctx).Equal(dst.ModTime(ctx)), "mutating dst should not mutate src -- is it Copying by pointer?") + } + } + + // Delete copy + err = dst.Remove(ctx) + require.NoError(t, err) + + // Test that server side copying files does the correct thing with metadata + t.Run("Metadata", func(t *testing.T) { + if !f.Features().WriteMetadata { + t.Skip("Skipping test as can't write metadata") + } + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + + // Create file with metadata + const srcName = "test metadata copy.txt" + const dstName = "test metadata copied.txt" + t1 := fstest.Time("2003-02-03T04:05:06.499999999Z") + t2 := fstest.Time("2004-03-03T04:05:06.499999999Z") + contents := random.String(100) + fileSrc := fstest.NewItem(srcName, contents, t1) + var testMetadata = fs.Metadata{ + // System metadata supported by all backends + "mtime": t1.Format(time.RFC3339Nano), + // User metadata + "potato": "jersey", + } + oSrc := PutTestContentsMetadata(ctx, t, f, &fileSrc, false, contents, true, "text/plain", testMetadata) + fstest.CheckEntryMetadata(ctx, t, f, oSrc, testMetadata) + + // Copy it with --metadata-set + ci.MetadataSet = fs.Metadata{ + // System metadata supported by all backends + "mtime": t2.Format(time.RFC3339Nano), + // User metadata + "potato": "royal", + } + oDst, err := doCopy(ctx, oSrc, dstName) + require.NoError(t, err) + fileDst := fileSrc + fileDst.Path = dstName + fileDst.ModTime = t2 + fstest.CheckListing(t, f, []fstest.Item{file1, file2, fileSrc, fileDst}) + + // Check metadata is correct + fstest.CheckEntryMetadata(ctx, t, f, oDst, ci.MetadataSet) + oDst = fstest.NewObject(ctx, t, f, dstName) + fstest.CheckEntryMetadata(ctx, t, f, oDst, ci.MetadataSet) + + // Remove test files + require.NoError(t, oSrc.Remove(ctx)) + require.NoError(t, oDst.Remove(ctx)) + }) + }) + + // TestFsMove tests Move + t.Run("FsMove", func(t *testing.T) { + skipIfNotOk(t) + + // Check have Move + doMove := f.Features().Move + if doMove == nil { + t.Skip("FS has no Mover interface") + } + + // state of files now: + // 1: file name.txt + // 2: hello sausage?/../z.txt + + var file1Move = file1 + var file2Move = file2 + + // check happy path, i.e. no naming conflicts when rename and move are two + // separate operations + file2Move.Path = "other.txt" + src := fstest.NewObject(ctx, t, f, file2.Path) + dst, err := doMove(ctx, src, file2Move.Path) + if err == fs.ErrorCantMove { + t.Skip("FS can't move") + } + require.NoError(t, err) + // check file exists in new listing + fstest.CheckListing(t, f, []fstest.Item{file1, file2Move}) + // Check dst lightly - list above has checked ModTime/Hashes + assert.Equal(t, file2Move.Path, dst.Remote()) + // 1: file name.txt + // 2: other.txt + + // Check conflict on "rename, then move" + file1Move.Path = "moveTest/other.txt" + src = fstest.NewObject(ctx, t, f, file1.Path) + _, err = doMove(ctx, src, file1Move.Path) + require.NoError(t, err) + fstest.CheckListing(t, f, []fstest.Item{file1Move, file2Move}) + // 1: moveTest/other.txt + // 2: other.txt + + // Check conflict on "move, then rename" + src = fstest.NewObject(ctx, t, f, file1Move.Path) + _, err = doMove(ctx, src, file1.Path) + require.NoError(t, err) + fstest.CheckListing(t, f, []fstest.Item{file1, file2Move}) + // 1: file name.txt + // 2: other.txt + + src = fstest.NewObject(ctx, t, f, file2Move.Path) + _, err = doMove(ctx, src, file2.Path) + require.NoError(t, err) + fstest.CheckListing(t, f, []fstest.Item{file1, file2}) + // 1: file name.txt + // 2: hello sausage?/../z.txt + + // Tidy up moveTest directory + require.NoError(t, f.Rmdir(ctx, "moveTest")) + + // Test that server side moving files does the correct thing with metadata + t.Run("Metadata", func(t *testing.T) { + if !f.Features().WriteMetadata { + t.Skip("Skipping test as can't write metadata") + } + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + + // Create file with metadata + const name = "test metadata move.txt" + const newName = "test metadata moved.txt" + t1 := fstest.Time("2003-02-03T04:05:06.499999999Z") + t2 := fstest.Time("2004-03-03T04:05:06.499999999Z") + file := fstest.NewItem(name, name, t1) + contents := random.String(100) + var testMetadata = fs.Metadata{ + // System metadata supported by all backends + "mtime": t1.Format(time.RFC3339Nano), + // User metadata + "potato": "jersey", + } + o := PutTestContentsMetadata(ctx, t, f, &file, false, contents, true, "text/plain", testMetadata) + fstest.CheckEntryMetadata(ctx, t, f, o, testMetadata) + + // Move it with --metadata-set + ci.MetadataSet = fs.Metadata{ + // System metadata supported by all backends + "mtime": t2.Format(time.RFC3339Nano), + // User metadata + "potato": "royal", + } + newO, err := doMove(ctx, o, newName) + require.NoError(t, err) + file.Path = newName + file.ModTime = t2 + fstest.CheckListing(t, f, []fstest.Item{file1, file2, file}) + + // Check metadata is correct + fstest.CheckEntryMetadata(ctx, t, f, newO, ci.MetadataSet) + newO = fstest.NewObject(ctx, t, f, newName) + fstest.CheckEntryMetadata(ctx, t, f, newO, ci.MetadataSet) + + // Remove test file + require.NoError(t, newO.Remove(ctx)) + }) + }) + + // Move src to this remote using server-side move operations. + // + // Will only be called if src.Fs().Name() == f.Name() + // + // If it isn't possible then return fs.ErrorCantDirMove + // + // If destination exists then return fs.ErrorDirExists + + // TestFsDirMove tests DirMove + // + // go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|FsDirMove)$ + t.Run("FsDirMove", func(t *testing.T) { + skipIfNotOk(t) + + // Check have DirMove + doDirMove := f.Features().DirMove + if doDirMove == nil { + t.Skip("FS has no DirMover interface") + } + + // Check it can't move onto itself + err := doDirMove(ctx, f, "", "") + require.Equal(t, fs.ErrorDirExists, err) + + // new remote + newRemote, _, removeNewRemote, err := fstest.RandomRemote() + require.NoError(t, err) + defer removeNewRemote() + + const newName = "new_name/sub_new_name" + // try the move + err = newRemote.Features().DirMove(ctx, f, "", newName) + require.NoError(t, err) + + // check remotes + // remote should not exist here + _, err = f.List(ctx, "") + assert.True(t, errors.Is(err, fs.ErrorDirNotFound)) + //fstest.CheckListingWithPrecision(t, remote, []fstest.Item{}, []string{}, remote.Precision()) + file1Copy := file1 + file1Copy.Path = path.Join(newName, file1.Path) + file2Copy := file2 + file2Copy.Path = path.Join(newName, file2.Path) + fstest.CheckListingWithPrecision(t, newRemote, []fstest.Item{file2Copy, file1Copy}, []string{ + "new_name", + "new_name/sub_new_name", + "new_name/sub_new_name/hello? sausage", + "new_name/sub_new_name/hello? sausage/ĂªĂ©", + "new_name/sub_new_name/hello? sausage/ĂªĂ©/Hello, 世界", + "new_name/sub_new_name/hello? sausage/ĂªĂ©/Hello, 世界/ \" ' @ < > & ? + ≠", + }, newRemote.Precision()) + + // move it back + err = doDirMove(ctx, newRemote, newName, "") + require.NoError(t, err) + + // check remotes + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file2, file1}, []string{ + "hello? sausage", + "hello? sausage/ĂªĂ©", + "hello? sausage/ĂªĂ©/Hello, 世界", + "hello? sausage/ĂªĂ©/Hello, 世界/ \" ' @ < > & ? + ≠", + }, f.Precision()) + fstest.CheckListingWithPrecision(t, newRemote, []fstest.Item{}, []string{ + "new_name", + }, newRemote.Precision()) + }) + + // TestFsRmdirFull tests removing a non empty directory + t.Run("FsRmdirFull", func(t *testing.T) { + skipIfNotOk(t) + if isBucketBasedButNotRoot(f) { + t.Skip("Skipping test as non root bucket-based remote") + } + err := f.Rmdir(ctx, "") + require.Error(t, err, "Expecting error on RMdir on non empty remote") + }) + + // TestFsPrecision tests the Precision of the Fs + t.Run("FsPrecision", func(t *testing.T) { + skipIfNotOk(t) + precision := f.Precision() + if precision == fs.ModTimeNotSupported { + return + } + if precision > time.Second || precision < 0 { + t.Fatalf("Precision out of range %v", precision) + } + // FIXME check expected precision + }) + + // TestObjectString tests the Object String method + t.Run("ObjectString", func(t *testing.T) { + skipIfNotOk(t) + obj := fstest.NewObject(ctx, t, f, file1.Path) + assert.Equal(t, file1.Path, obj.String()) + if opt.NilObject != nil { + assert.Equal(t, "<nil>", opt.NilObject.String()) + } + }) + + // TestObjectFs tests the object can be found + t.Run("ObjectFs", func(t *testing.T) { + skipIfNotOk(t) + obj := fstest.NewObject(ctx, t, f, file1.Path) + // If this is set we don't do the direct comparison of + // the Fs from the object as it may be different + if opt.SkipFsMatch { + return + } + testRemote := f + if obj.Fs() != testRemote { + // Check to see if this wraps something else + if doUnWrap := testRemote.Features().UnWrap; doUnWrap != nil { + testRemote = doUnWrap() + } + } + assert.Equal(t, obj.Fs(), testRemote) + }) + + // TestObjectRemote tests the Remote is correct + t.Run("ObjectRemote", func(t *testing.T) { + skipIfNotOk(t) + obj := fstest.NewObject(ctx, t, f, file1.Path) + assert.Equal(t, file1.Path, obj.Remote()) + }) + + // TestObjectHashes checks all the hashes the object supports + t.Run("ObjectHashes", func(t *testing.T) { + skipIfNotOk(t) + obj := fstest.NewObject(ctx, t, f, file1.Path) + file1.CheckHashes(t, obj) + }) + + // TestObjectModTime tests the ModTime of the object is correct + TestObjectModTime := func(t *testing.T) { + skipIfNotOk(t) + obj := fstest.NewObject(ctx, t, f, file1.Path) + file1.CheckModTime(t, obj, obj.ModTime(ctx), f.Precision()) + } + t.Run("ObjectModTime", TestObjectModTime) + + // TestObjectMimeType tests the MimeType of the object is correct + t.Run("ObjectMimeType", func(t *testing.T) { + skipIfNotOk(t) + features := f.Features() + obj := fstest.NewObject(ctx, t, f, file1.Path) + do, ok := obj.(fs.MimeTyper) + if !ok { + require.False(t, features.ReadMimeType, "Features.ReadMimeType is set but Object.MimeType method not found") + t.Skip("MimeType method not supported") + } + mimeType := do.MimeType(ctx) + if !features.ReadMimeType { + require.Equal(t, "", mimeType, "Features.ReadMimeType is not set but Object.MimeType returned a non-empty MimeType") + } else if features.WriteMimeType { + assert.Equal(t, file1MimeType, mimeType, "can read and write mime types but failed") + } else { + if strings.ContainsRune(mimeType, ';') { + assert.Equal(t, "text/plain; charset=utf-8", mimeType) + } else { + assert.Equal(t, "text/plain", mimeType) + } + } + }) + + // TestObjectMetadata tests the Metadata of the object is correct + t.Run("ObjectMetadata", func(t *testing.T) { + skipIfNotOk(t) + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + features := f.Features() + obj := fstest.NewObject(ctx, t, f, file1.Path) + do, objectHasMetadata := obj.(fs.Metadataer) + if objectHasMetadata || features.ReadMetadata || features.WriteMetadata || features.UserMetadata { + fsInfo := fs.FindFromFs(f) + require.NotNil(t, fsInfo) + require.NotNil(t, fsInfo.MetadataInfo, "Object declares metadata support but no MetadataInfo in RegInfo") + } + if !objectHasMetadata { + require.False(t, features.ReadMetadata, "Features.ReadMetadata is set but Object.Metadata method not found") + t.Skip("Metadata method not supported") + } + metadata, err := do.Metadata(ctx) + require.NoError(t, err) + // check standard metadata + for k, v := range metadata { + switch k { + case "atime", "btime", "mtime": + mtime, err := time.Parse(time.RFC3339Nano, v) + require.NoError(t, err) + if k == "mtime" { + fstest.AssertTimeEqualWithPrecision(t, file1.Path, file1.ModTime, mtime, f.Precision()) + } + } + } + if !features.ReadMetadata { + if metadata != nil && !features.Overlay { + require.Equal(t, "", metadata, "Features.ReadMetadata is not set but Object.Metadata returned a non nil Metadata: %#v", metadata) + } + } else if features.WriteMetadata { + require.NotNil(t, metadata) + if features.UserMetadata { + // check all the metadata bits we uploaded are present - there may be more we didn't write + for k, v := range file1Metadata { + assert.Equal(t, v, metadata[k], "can read and write metadata but failed on key %q (want=%+v, got=%+v)", k, file1Metadata, metadata) + } + } + // Now test we can set the mtime and content-type via the metadata and these take precedence + t.Run("mtime", func(t *testing.T) { + path := "metadatatest" + mtimeModTime := fstest.Time("2002-02-03T04:05:06.499999999Z") + modTime := fstest.Time("2003-02-03T04:05:06.499999999Z") + item := fstest.NewItem(path, path, modTime) + metaMimeType := "application/zip" + mimeType := "application/gzip" + metadata := fs.Metadata{ + "mtime": mtimeModTime.Format(time.RFC3339Nano), + "content-type": metaMimeType, + } + // This checks the mtime is correct also and returns the re-read object + _, obj := testPutMimeType(ctx, t, f, &item, mimeType, metadata) + defer func() { + assert.NoError(t, obj.Remove(ctx)) + }() + // Check content-type got updated too + if features.ReadMimeType && features.WriteMimeType { + // read the object from scratch + o, err := f.NewObject(ctx, path) + require.NoError(t, err) + + // Check the mimetype is correct + do, ok := o.(fs.MimeTyper) + require.True(t, ok) + gotMimeType := do.MimeType(ctx) + assert.Equal(t, metaMimeType, gotMimeType) + } + }) + } // else: Have some metadata here we didn't write - can't really check it! + }) + + // TestObjectSetMetadata tests the SetMetadata of the object + t.Run("ObjectSetMetadata", func(t *testing.T) { + skipIfNotOk(t) + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + features := f.Features() + + // Test to see if SetMetadata is supported on an existing object before creating a new one + obj := fstest.NewObject(ctx, t, f, file1.Path) + _, objectHasSetMetadata := obj.(fs.SetMetadataer) + if !objectHasSetMetadata { + t.Skip("SetMetadata method not supported") + } + if !features.Overlay { + require.True(t, features.WriteMetadata, "Features.WriteMetadata is false but Object.SetMetadata found") + } + if !features.ReadMetadata { + t.Skip("SetMetadata can't be tested without ReadMetadata") + } + + // Create file with metadata + const fileName = "test set metadata.txt" + t1 := fstest.Time("2003-02-03T04:05:06.499999999Z") + t2 := fstest.Time("2004-03-03T04:05:06.499999999Z") + contents := random.String(100) + file := fstest.NewItem(fileName, contents, t1) + var testMetadata = fs.Metadata{ + // System metadata supported by all backends + "mtime": t1.Format(time.RFC3339Nano), + // User metadata + "potato": "jersey", + } + obj = PutTestContentsMetadata(ctx, t, f, &file, true, contents, true, "text/plain", testMetadata) + fstest.CheckEntryMetadata(ctx, t, f, obj, testMetadata) + do, objectHasSetMetadata := obj.(fs.SetMetadataer) + require.True(t, objectHasSetMetadata) + + // Set new metadata + err := do.SetMetadata(ctx, fs.Metadata{ + // System metadata supported by all backends + "mtime": t2.Format(time.RFC3339Nano), + // User metadata + "potato": "royal", + }) + if err == fs.ErrorNotImplemented { + t.Log("SetMetadata returned fs.ErrorNotImplemented") + } else { + require.NoError(t, err) + file.ModTime = t2 + fstest.CheckListing(t, f, []fstest.Item{file1, file2, file}) + + // Check metadata is correct + fstest.CheckEntryMetadata(ctx, t, f, obj, ci.MetadataSet) + obj = fstest.NewObject(ctx, t, f, fileName) + fstest.CheckEntryMetadata(ctx, t, f, obj, ci.MetadataSet) + } + + // Remove test file + require.NoError(t, obj.Remove(ctx)) + }) + + // TestObjectSetModTime tests that SetModTime works + t.Run("ObjectSetModTime", func(t *testing.T) { + skipIfNotOk(t) + newModTime := fstest.Time("2011-12-13T14:15:16.999999999Z") + obj := fstest.NewObject(ctx, t, f, file1.Path) + err := obj.SetModTime(ctx, newModTime) + if err == fs.ErrorCantSetModTime || err == fs.ErrorCantSetModTimeWithoutDelete { + t.Log(err) + return + } + require.NoError(t, err) + file1.ModTime = newModTime + file1.CheckModTime(t, obj, obj.ModTime(ctx), f.Precision()) + // And make a new object and read it from there too + TestObjectModTime(t) + }) + + // TestObjectSize tests that Size works + t.Run("ObjectSize", func(t *testing.T) { + skipIfNotOk(t) + obj := fstest.NewObject(ctx, t, f, file1.Path) + assert.Equal(t, file1.Size, obj.Size()) + }) + + // TestObjectOpen tests that Open works + t.Run("ObjectOpen", func(t *testing.T) { + skipIfNotOk(t) + obj := fstest.NewObject(ctx, t, f, file1.Path) + assert.Equal(t, file1Contents, ReadObject(ctx, t, obj, -1), "contents of file1 differ") + }) + + // TestObjectOpenSeek tests that Open works with SeekOption + t.Run("ObjectOpenSeek", func(t *testing.T) { + skipIfNotOk(t) + obj := fstest.NewObject(ctx, t, f, file1.Path) + assert.Equal(t, file1Contents[50:], ReadObject(ctx, t, obj, -1, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek") + }) + + // TestObjectOpenRange tests that Open works with RangeOption + // + // go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|ObjectOpenRange)$' + t.Run("ObjectOpenRange", func(t *testing.T) { + skipIfNotOk(t) + obj := fstest.NewObject(ctx, t, f, file1.Path) + for _, test := range []struct { + ro fs.RangeOption + wantStart, wantEnd int + }{ + {fs.RangeOption{Start: 5, End: 15}, 5, 16}, + {fs.RangeOption{Start: 80, End: -1}, 80, 100}, + {fs.RangeOption{Start: 81, End: 100000}, 81, 100}, + {fs.RangeOption{Start: -1, End: 20}, 80, 100}, // if start is omitted this means get the final bytes + // {fs.RangeOption{Start: -1, End: -1}, 0, 100}, - this seems to work but the RFC doesn't define it + } { + got := ReadObject(ctx, t, obj, -1, &test.ro) + foundAt := strings.Index(file1Contents, got) + help := fmt.Sprintf("%#v failed want [%d:%d] got [%d:%d]", test.ro, test.wantStart, test.wantEnd, foundAt, foundAt+len(got)) + assert.Equal(t, file1Contents[test.wantStart:test.wantEnd], got, help) + } + }) + + // TestObjectPartialRead tests that reading only part of the object does the correct thing + t.Run("ObjectPartialRead", func(t *testing.T) { + skipIfNotOk(t) + obj := fstest.NewObject(ctx, t, f, file1.Path) + assert.Equal(t, file1Contents[:50], ReadObject(ctx, t, obj, 50), "contents of file1 differ after limited read") + }) + + // TestObjectUpdate tests that Update works + t.Run("ObjectUpdate", func(t *testing.T) { + skipIfNotOk(t) + contents := random.String(200) + var h *hash.MultiHasher + + file1.Size = int64(len(contents)) + obj := fstest.NewObject(ctx, t, f, file1.Path) + remoteBefore := obj.Remote() + obji := object.NewStaticObjectInfo(file1.Path+"-should-be-ignored.bin", file1.ModTime, int64(len(contents)), true, nil, obj.Fs()) + retry(t, "Update object", func() error { + buf := bytes.NewBufferString(contents) + h = hash.NewMultiHasher() + in := io.TeeReader(buf, h) + return obj.Update(ctx, in, obji) + }) + remoteAfter := obj.Remote() + assert.Equal(t, remoteBefore, remoteAfter, "Remote should not change") + file1.Hashes = h.Sums() + + // check the object has been updated + file1.Check(t, obj, f.Precision()) + + // Re-read the object and check again + obj = fstest.NewObject(ctx, t, f, file1.Path) + file1.Check(t, obj, f.Precision()) + + // check contents correct + assert.Equal(t, contents, ReadObject(ctx, t, obj, -1), "contents of updated file1 differ") + file1Contents = contents + }) + + // TestObjectStorable tests that Storable works + t.Run("ObjectStorable", func(t *testing.T) { + skipIfNotOk(t) + obj := fstest.NewObject(ctx, t, f, file1.Path) + require.NotNil(t, !obj.Storable(), "Expecting object to be storable") + }) + + // TestFsIsFile tests that an error is returned along with a valid fs + // which points to the parent directory. + t.Run("FsIsFile", func(t *testing.T) { + skipIfNotOk(t) + remoteName := subRemoteName + "/" + file2.Path + file2Copy := file2 + file2Copy.Path = "z.txt" + fileRemote, err := fs.NewFs(context.Background(), remoteName) + require.NotNil(t, fileRemote) + assert.Equal(t, fs.ErrorIsFile, err) + + // Check Fs.Root returns the right thing + t.Run("FsRoot", func(t *testing.T) { + skipIfNotOk(t) + got := fileRemote.Root() + remoteDir := path.Dir(remoteName) + want := remoteDir + colon := strings.LastIndex(want, ":") + if colon >= 0 { + want = want[colon+1:] + } + if isLocalRemote { + // only check last path element on local + require.Equal(t, filepath.Base(remoteDir), filepath.Base(got)) + } else { + require.Equal(t, want, got) + } + }) + + if strings.HasPrefix(remoteName, "TestChunker") && strings.Contains(remoteName, "Nometa") { + // TODO fix chunker and remove this bypass + t.Logf("Skip listing check -- chunker can't yet handle this tricky case") + return + } + fstest.CheckListing(t, fileRemote, []fstest.Item{file2Copy}) + }) + + // TestFsIsFileNotFound tests that an error is not returned if no object is found + t.Run("FsIsFileNotFound", func(t *testing.T) { + skipIfNotOk(t) + remoteName := subRemoteName + "/not found.txt" + fileRemote, err := fs.NewFs(context.Background(), remoteName) + require.NoError(t, err) + fstest.CheckListing(t, fileRemote, []fstest.Item{}) + }) + + // Test that things work from the root + t.Run("FromRoot", func(t *testing.T) { + if features := f.Features(); features.BucketBased && !features.BucketBasedRootOK { + t.Skip("Can't list from root on this remote") + } + + parsed, err := fspath.Parse(subRemoteName) + require.NoError(t, err) + configName, configLeaf := parsed.ConfigString, parsed.Path + if configName == "" { + configName, configLeaf = path.Split(subRemoteName) + } else { + configName += ":" + } + t.Logf("Opening root remote %q path %q from %q", configName, configLeaf, subRemoteName) + rootRemote, err := fs.NewFs(context.Background(), configName) + if errors.Is(err, fs.ErrorCantListRoot) { + t.Skip("Can't list from root on this remote") + } + require.NoError(t, err) + + file1Root := file1 + file1Root.Path = path.Join(configLeaf, file1Root.Path) + file2Root := file2 + file2Root.Path = path.Join(configLeaf, file2Root.Path) + var dirs []string + dir := file2.Path + for { + dir = path.Dir(dir) + if dir == "" || dir == "." || dir == "/" { + break + } + dirs = append(dirs, path.Join(configLeaf, dir)) + } + + // Check that we can see file1 and file2 from the root + t.Run("List", func(t *testing.T) { + fstest.CheckListingWithRoot(t, rootRemote, configLeaf, []fstest.Item{file1Root, file2Root}, dirs, rootRemote.Precision()) + }) + + // Check that listing the entries is OK + t.Run("ListEntries", func(t *testing.T) { + entries, err := rootRemote.List(context.Background(), configLeaf) + require.NoError(t, err) + fstest.CompareItems(t, entries, []fstest.Item{file1Root}, dirs[len(dirs)-1:], rootRemote.Precision(), "ListEntries") + }) + + // List the root with ListR + t.Run("ListR", func(t *testing.T) { + doListR := rootRemote.Features().ListR + if doListR == nil { + t.Skip("FS has no ListR interface") + } + file1Found, file2Found := false, false + stopTime := time.Now().Add(10 * time.Second) + errTooMany := errors.New("too many files") + errFound := errors.New("found") + err := doListR(context.Background(), "", func(entries fs.DirEntries) error { + for _, entry := range entries { + remote := entry.Remote() + if remote == file1Root.Path { + file1Found = true + } + if remote == file2Root.Path { + file2Found = true + } + if file1Found && file2Found { + return errFound + } + } + if time.Now().After(stopTime) { + return errTooMany + } + return nil + }) + if !errors.Is(err, errFound) && !errors.Is(err, errTooMany) { + assert.NoError(t, err) + } + if !errors.Is(err, errTooMany) { + assert.True(t, file1Found, "file1Root %q not found", file1Root.Path) + assert.True(t, file2Found, "file2Root %q not found", file2Root.Path) + } else { + t.Logf("Too many files to list - giving up") + } + }) + + // Create a new file + t.Run("Put", func(t *testing.T) { + file3Root := fstest.Item{ + ModTime: time.Now(), + Path: path.Join(configLeaf, "created from root.txt"), + } + _, file3Obj := testPut(ctx, t, rootRemote, &file3Root) + fstest.CheckListingWithRoot(t, rootRemote, configLeaf, []fstest.Item{file1Root, file2Root, file3Root}, nil, rootRemote.Precision()) + + // And then remove it + t.Run("Remove", func(t *testing.T) { + require.NoError(t, file3Obj.Remove(context.Background())) + fstest.CheckListingWithRoot(t, rootRemote, configLeaf, []fstest.Item{file1Root, file2Root}, nil, rootRemote.Precision()) + }) + }) + }) + + // TestPublicLink tests creation of sharable, public links + // go test -v -run 'TestIntegration/Test(Setup|Init|FsMkdir|FsPutFile1|FsPutFile2|FsUpdateFile1|PublicLink)$' + t.Run("PublicLink", func(t *testing.T) { + skipIfNotOk(t) + + publicLinkFunc := f.Features().PublicLink + if publicLinkFunc == nil { + t.Skip("FS has no PublicLinker interface") + } + + type PublicLinkFunc func(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) + wrapPublicLinkFunc := func(f PublicLinkFunc) PublicLinkFunc { + return func(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) { + link, err = publicLinkFunc(ctx, remote, expire, unlink) + if err == nil { + return + } + // For OneDrive Personal, link expiry is a premium feature + // Don't let it fail the test (https://github.com/rclone/rclone/issues/5420) + if fsInfo.Name == "onedrive" && strings.Contains(err.Error(), "accountUpgradeRequired") { + t.Log("treating accountUpgradeRequired as success for PublicLink") + link, err = "bogus link to "+remote, nil + } + return + } + } + + expiry := fs.Duration(120 * time.Second) + doPublicLink := wrapPublicLinkFunc(publicLinkFunc) + + // if object not found + link, err := doPublicLink(ctx, file1.Path+"_does_not_exist", expiry, false) + require.Error(t, err, "Expected to get error when file doesn't exist") + require.Equal(t, "", link, "Expected link to be empty on error") + + // sharing file for the first time + link1, err := doPublicLink(ctx, file1.Path, expiry, false) + require.NoError(t, err) + require.NotEqual(t, "", link1, "Link should not be empty") + + link2, err := doPublicLink(ctx, file2.Path, expiry, false) + require.NoError(t, err) + require.NotEqual(t, "", link2, "Link should not be empty") + + require.NotEqual(t, link1, link2, "Links to different files should differ") + + // sharing file for the 2nd time + link1, err = doPublicLink(ctx, file1.Path, expiry, false) + require.NoError(t, err) + require.NotEqual(t, "", link1, "Link should not be empty") + + // sharing directory for the first time + path := path.Dir(file2.Path) + link3, err := doPublicLink(ctx, path, expiry, false) + if err != nil && (errors.Is(err, fs.ErrorCantShareDirectories) || errors.Is(err, fs.ErrorObjectNotFound)) { + t.Log("skipping directory tests as not supported on this backend") + } else { + require.NoError(t, err) + require.NotEqual(t, "", link3, "Link should not be empty") + + // sharing directory for the second time + link3, err = doPublicLink(ctx, path, expiry, false) + require.NoError(t, err) + require.NotEqual(t, "", link3, "Link should not be empty") + + // sharing the "root" directory in a subremote + subRemote, _, removeSubRemote, err := fstest.RandomRemote() + require.NoError(t, err) + defer removeSubRemote() + // ensure sub remote isn't empty + buf := bytes.NewBufferString("somecontent") + obji := object.NewStaticObjectInfo("somefile", time.Now(), int64(buf.Len()), true, nil, nil) + retry(t, "Put", func() error { + _, err := subRemote.Put(ctx, buf, obji) + return err + }) + + link4, err := wrapPublicLinkFunc(subRemote.Features().PublicLink)(ctx, "", expiry, false) + require.NoError(t, err, "Sharing root in a sub-remote should work") + require.NotEqual(t, "", link4, "Link should not be empty") + } + }) + + // TestSetTier tests SetTier and GetTier functionality + t.Run("SetTier", func(t *testing.T) { + skipIfNotSetTier(t) + obj := fstest.NewObject(ctx, t, f, file1.Path) + setter, ok := obj.(fs.SetTierer) + assert.NotNil(t, ok) + getter, ok := obj.(fs.GetTierer) + assert.NotNil(t, ok) + // If interfaces are supported TiersToTest should contain + // at least one entry + supportedTiers := opt.TiersToTest + assert.NotEmpty(t, supportedTiers) + // test set tier changes on supported storage classes or tiers + for _, tier := range supportedTiers { + err := setter.SetTier(tier) + assert.Nil(t, err) + got := getter.GetTier() + assert.Equal(t, tier, got) + } + }) + + // Check to see if Fs that wrap other Objects implement all the optional methods + t.Run("ObjectCheckWrap", func(t *testing.T) { + skipIfNotOk(t) + if opt.SkipObjectCheckWrap { + t.Skip("Skipping FsCheckWrap on this Fs") + } + ft := new(fs.Features).Fill(ctx, f) + if ft.UnWrap == nil { + t.Skip("Not a wrapping Fs") + } + obj := fstest.NewObject(ctx, t, f, file1.Path) + _, unsupported := fs.ObjectOptionalInterfaces(obj) + for _, name := range unsupported { + if !stringsContains(name, opt.UnimplementableObjectMethods) { + t.Errorf("Missing Object wrapper for %s", name) + } + } + }) + + // Run tests for bucket based Fs + // TestIntegration/FsMkdir/FsPutFiles/Bucket + t.Run("Bucket", func(t *testing.T) { + // Test if this Fs is bucket based - this test won't work for wrapped bucket based backends. + if !f.Features().BucketBased { + t.Skip("Not a bucket based backend") + } + if f.Features().CanHaveEmptyDirectories { + t.Skip("Can have empty directories") + } + if !f.Features().DoubleSlash { + t.Skip("Can't have // in paths") + } + // Create some troublesome file names + fileNames := []string{ + file1.Path, + file2.Path, + ".leadingdot", + "/.leadingdot", + "///tripleslash", + "//doubleslash", + "dir/.leadingdot", + "dir///tripleslash", + "dir//doubleslash", + } + dirNames := []string{ + "hello? sausage", + "hello? sausage/ĂªĂ©", + "hello? sausage/ĂªĂ©/Hello, 世界", + "hello? sausage/ĂªĂ©/Hello, 世界/ \" ' @ < > & ? + ≠", + "/", + "//", + "///", + "dir", + "dir/", + "dir//", + } + t1 := fstest.Time("2003-02-03T04:05:06.499999999Z") + var objs []fs.Object + for _, fileName := range fileNames[2:] { + contents := "bad file name: " + fileName + file := fstest.NewItem(fileName, contents, t1) + objs = append(objs, PutTestContents(ctx, t, f, &file, contents, true)) + } + + // Check they arrived + // This uses walk.Walk with a max size set to make sure we don't use ListR + check := func(f fs.Fs, dir string, wantFileNames, wantDirNames []string) { + t.Helper() + var gotFileNames, gotDirNames []string + require.NoError(t, walk.Walk(ctx, f, dir, true, 100, func(path string, entries fs.DirEntries, err error) error { + if err != nil { + return err + } + for _, entry := range entries { + if _, isObj := entry.(fs.Object); isObj { + gotFileNames = append(gotFileNames, entry.Remote()) + } else { + gotDirNames = append(gotDirNames, entry.Remote()) + } + } + return nil + })) + sort.Strings(wantDirNames) + sort.Strings(wantFileNames) + sort.Strings(gotDirNames) + sort.Strings(gotFileNames) + assert.Equal(t, wantFileNames, gotFileNames) + assert.Equal(t, wantDirNames, gotDirNames) + } + check(f, "", fileNames, dirNames) + check(f, "/", []string{ + "/.leadingdot", + "///tripleslash", + "//doubleslash", + }, []string{ + "//", + "///", + }) + check(f, "//", []string{ + "///tripleslash", + "//doubleslash", + }, []string{ + "///", + }) + check(f, "dir", []string{ + "dir/.leadingdot", + "dir///tripleslash", + "dir//doubleslash", + }, []string{ + "dir/", + "dir//", + }) + check(f, "dir/", []string{ + "dir///tripleslash", + "dir//doubleslash", + }, []string{ + "dir//", + }) + check(f, "dir//", []string{ + "dir///tripleslash", + }, nil) + + // Now create a backend not at the root of a bucket + f2, err := fs.NewFs(ctx, subRemoteName+"/dir") + require.NoError(t, err) + check(f2, "", []string{ + ".leadingdot", + "//tripleslash", + "/doubleslash", + }, []string{ + "/", + "//", + }) + check(f2, "/", []string{ + "//tripleslash", + "/doubleslash", + }, []string{ + "//", + }) + check(f2, "//", []string{ + "//tripleslash", + }, []string(nil)) + + // Remove the objects + for _, obj := range objs { + assert.NoError(t, obj.Remove(ctx)) + } + + // Check they are gone + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file1, file2}, []string{ + "hello? sausage", + "hello? sausage/ĂªĂ©", + "hello? sausage/ĂªĂ©/Hello, 世界", + "hello? sausage/ĂªĂ©/Hello, 世界/ \" ' @ < > & ? + ≠", + }, fs.GetModifyWindow(ctx, f)) + }) + + // State of remote at the moment the internal tests are called + InternalTestFiles = []fstest.Item{file1, file2} + + // TestObjectRemove tests Remove + t.Run("ObjectRemove", func(t *testing.T) { + skipIfNotOk(t) + // remove file1 + obj := fstest.NewObject(ctx, t, f, file1.Path) + err := obj.Remove(ctx) + require.NoError(t, err) + // check listing without modtime as TestPublicLink may change the modtime + fstest.CheckListingWithPrecision(t, f, []fstest.Item{file2}, nil, fs.ModTimeNotSupported) + // Show the internal tests file2 is gone + InternalTestFiles = []fstest.Item{file2} + }) + + // TestAbout tests the About optional interface + t.Run("ObjectAbout", func(t *testing.T) { + skipIfNotOk(t) + + // Check have About + doAbout := f.Features().About + if doAbout == nil { + t.Skip("FS does not support About") + } + + // Can't really check the output much! + usage, err := doAbout(context.Background()) + require.NoError(t, err) + require.NotNil(t, usage) + assert.NotEqual(t, int64(0), usage.Total) + }) + + // Just file2 remains for Purge to clean up + + // TestFsPutStream tests uploading files when size isn't known in advance. + // This may trigger large buffer allocation in some backends, keep it + // close to the end of suite. (See fs/operations/xtra_operations_test.go) + t.Run("FsPutStream", func(t *testing.T) { + skipIfNotOk(t) + if f.Features().PutStream == nil { + t.Skip("FS has no PutStream interface") + } + + for _, contentSize := range []int{0, 100} { + t.Run(strconv.Itoa(contentSize), func(t *testing.T) { + file := fstest.Item{ + ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), + Path: "piped data.txt", + Size: -1, // use unknown size during upload + } + + var ( + err error + obj fs.Object + uploadHash *hash.MultiHasher + ) + retry(t, "PutStream", func() error { + contents := random.String(contentSize) + buf := bytes.NewBufferString(contents) + uploadHash = hash.NewMultiHasher() + in := io.TeeReader(buf, uploadHash) + + file.Size = -1 + obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil) + obj, err = f.Features().PutStream(ctx, in, obji) + return err + }) + file.Hashes = uploadHash.Sums() + file.Size = int64(contentSize) // use correct size when checking + file.Check(t, obj, f.Precision()) + // Re-read the object and check again + obj = fstest.NewObject(ctx, t, f, file.Path) + file.Check(t, obj, f.Precision()) + require.NoError(t, obj.Remove(ctx)) + }) + } + }) + + // TestInternal calls InternalTest() on the Fs + t.Run("Internal", func(t *testing.T) { + skipIfNotOk(t) + if it, ok := f.(InternalTester); ok { + it.InternalTest(t) + } else { + t.Skipf("%T does not implement InternalTester", f) + } + }) + + }) + + // TestFsPutChunked may trigger large buffer allocation with + // some backends (see fs/operations/xtra_operations_test.go), + // keep it closer to the end of suite. + t.Run("FsPutChunked", func(t *testing.T) { + skipIfNotOk(t) + if testing.Short() { + t.Skip("not running with -short") + } + + if opt.ChunkedUpload.Skip { + t.Skip("skipping as ChunkedUpload.Skip is set") + } + + setUploadChunkSizer, _ := f.(SetUploadChunkSizer) + if setUploadChunkSizer == nil { + t.Skipf("%T does not implement SetUploadChunkSizer", f) + } + + setUploadCutoffer, _ := f.(SetUploadCutoffer) + + minChunkSize := max(opt.ChunkedUpload.MinChunkSize, 100) + if opt.ChunkedUpload.CeilChunkSize != nil { + minChunkSize = opt.ChunkedUpload.CeilChunkSize(minChunkSize) + } + + maxChunkSize := max(2*fs.Mebi, 2*minChunkSize) + if opt.ChunkedUpload.MaxChunkSize > 0 && maxChunkSize > opt.ChunkedUpload.MaxChunkSize { + maxChunkSize = opt.ChunkedUpload.MaxChunkSize + } + if opt.ChunkedUpload.CeilChunkSize != nil { + maxChunkSize = opt.ChunkedUpload.CeilChunkSize(maxChunkSize) + } + + next := func(f func(fs.SizeSuffix) fs.SizeSuffix) fs.SizeSuffix { + s := f(minChunkSize) + if s > maxChunkSize { + s = minChunkSize + } + return s + } + + chunkSizes := fs.SizeSuffixList{ + minChunkSize, + minChunkSize + (maxChunkSize-minChunkSize)/3, + next(NextPowerOfTwo), + next(NextMultipleOf(100000)), + next(NextMultipleOf(100001)), + maxChunkSize, + } + chunkSizes.Sort() + + // Set the minimum chunk size, upload cutoff and reset it at the end + oldChunkSize, err := setUploadChunkSizer.SetUploadChunkSize(minChunkSize) + require.NoError(t, err) + var oldUploadCutoff fs.SizeSuffix + if setUploadCutoffer != nil { + oldUploadCutoff, err = setUploadCutoffer.SetUploadCutoff(minChunkSize) + require.NoError(t, err) + } + defer func() { + _, err := setUploadChunkSizer.SetUploadChunkSize(oldChunkSize) + assert.NoError(t, err) + if setUploadCutoffer != nil { + _, err := setUploadCutoffer.SetUploadCutoff(oldUploadCutoff) + assert.NoError(t, err) + } + }() + + var lastCs fs.SizeSuffix + for _, cs := range chunkSizes { + if cs <= lastCs { + continue + } + if opt.ChunkedUpload.CeilChunkSize != nil { + cs = opt.ChunkedUpload.CeilChunkSize(cs) + } + lastCs = cs + + t.Run(cs.String(), func(t *testing.T) { + _, err := setUploadChunkSizer.SetUploadChunkSize(cs) + require.NoError(t, err) + if setUploadCutoffer != nil { + _, err = setUploadCutoffer.SetUploadCutoff(cs) + require.NoError(t, err) + } + + var testChunks []fs.SizeSuffix + if opt.ChunkedUpload.NeedMultipleChunks { + // If NeedMultipleChunks is set then test with > cs + testChunks = []fs.SizeSuffix{cs + 1, 2 * cs, 2*cs + 1} + } else { + testChunks = []fs.SizeSuffix{cs - 1, cs, 2*cs + 1} + } + + for _, fileSize := range testChunks { + t.Run(fmt.Sprintf("%d", fileSize), func(t *testing.T) { + TestPutLarge(ctx, t, f, &fstest.Item{ + ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), + Path: fmt.Sprintf("chunked-%s-%s.bin", cs.String(), fileSize.String()), + Size: int64(fileSize), + }) + t.Run("Streamed", func(t *testing.T) { + if f.Features().PutStream == nil { + t.Skip("FS has no PutStream interface") + } + TestPutLargeStreamed(ctx, t, f, &fstest.Item{ + ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), + Path: fmt.Sprintf("chunked-%s-%s-streamed.bin", cs.String(), fileSize.String()), + Size: int64(fileSize), + }) + }) + }) + } + }) + } + }) + + // Copy files with chunked copy if available + t.Run("FsCopyChunked", func(t *testing.T) { + skipIfNotOk(t) + if testing.Short() { + t.Skip("not running with -short") + } + + // Check have Copy + doCopy := f.Features().Copy + if doCopy == nil { + t.Skip("FS has no Copier interface") + } + + if opt.ChunkedUpload.Skip { + t.Skip("skipping as ChunkedUpload.Skip is set") + } + + do, _ := f.(SetCopyCutoffer) + if do == nil { + t.Skipf("%T does not implement SetCopyCutoff", f) + } + + minChunkSize := max(opt.ChunkedUpload.MinChunkSize, 100) + if opt.ChunkedUpload.CeilChunkSize != nil { + minChunkSize = opt.ChunkedUpload.CeilChunkSize(minChunkSize) + } + + // Test setting the copy cutoff before we get going + _, err := do.SetCopyCutoff(minChunkSize) + if errors.Is(err, fs.ErrorNotImplemented) { + t.Skipf("%T does not support SetCopyCutoff: %v", f, err) + } + require.NoError(t, err) + + chunkSizes := fs.SizeSuffixList{ + minChunkSize, + minChunkSize + 1, + 2*minChunkSize - 1, + 2 * minChunkSize, + 2*minChunkSize + 1, + } + for _, chunkSize := range chunkSizes { + t.Run(fmt.Sprintf("%d", chunkSize), func(t *testing.T) { + contents := random.String(int(chunkSize)) + item := fstest.NewItem("chunked-copy", contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) + src := PutTestContents(ctx, t, f, &item, contents, true) + defer func() { + assert.NoError(t, src.Remove(ctx)) + }() + + var itemCopy = item + itemCopy.Path += ".copy" + + // Set copy cutoff to minimum value so we make chunks + origCutoff, err := do.SetCopyCutoff(minChunkSize) + require.NoError(t, err) + defer func() { + _, err = do.SetCopyCutoff(origCutoff) + require.NoError(t, err) + }() + + // Do the copy + dst, err := doCopy(ctx, src, itemCopy.Path) + require.NoError(t, err) + defer func() { + assert.NoError(t, dst.Remove(ctx)) + }() + + // Check size + assert.Equal(t, src.Size(), dst.Size()) + + // Check modtime + srcModTime := src.ModTime(ctx) + dstModTime := dst.ModTime(ctx) + assert.True(t, srcModTime.Equal(dstModTime)) + + // Make sure contents are correct + gotContents := ReadObject(ctx, t, dst, -1) + assert.Equal(t, contents, gotContents) + }) + } + }) + + // TestFsUploadUnknownSize ensures Fs.Put() and Object.Update() don't panic when + // src.Size() == -1 + // + // This may trigger large buffer allocation in some backends, keep it + // closer to the suite end. (See fs/operations/xtra_operations_test.go) + t.Run("FsUploadUnknownSize", func(t *testing.T) { + skipIfNotOk(t) + + t.Run("FsPutUnknownSize", func(t *testing.T) { + defer func() { + assert.Nil(t, recover(), "Fs.Put() should not panic when src.Size() == -1") + }() + + contents := random.String(100) + in := bytes.NewBufferString(contents) + + obji := object.NewStaticObjectInfo("unknown-size-put.txt", fstest.Time("2002-02-03T04:05:06.499999999Z"), -1, true, nil, nil) + obj, err := f.Put(ctx, in, obji) + if err == nil { + require.NoError(t, obj.Remove(ctx), "successfully uploaded unknown-sized file but failed to remove") + } + // if err != nil: it's okay as long as no panic + }) + + t.Run("FsUpdateUnknownSize", func(t *testing.T) { + unknownSizeUpdateFile := fstest.Item{ + ModTime: fstest.Time("2002-02-03T04:05:06.499999999Z"), + Path: "unknown-size-update.txt", + } + + testPut(ctx, t, f, &unknownSizeUpdateFile) + + defer func() { + assert.Nil(t, recover(), "Object.Update() should not panic when src.Size() == -1") + }() + + newContents := random.String(200) + in := bytes.NewBufferString(newContents) + + obj := fstest.NewObject(ctx, t, f, unknownSizeUpdateFile.Path) + obji := object.NewStaticObjectInfo(unknownSizeUpdateFile.Path, unknownSizeUpdateFile.ModTime, -1, true, nil, obj.Fs()) + err := obj.Update(ctx, in, obji) + if err == nil { + require.NoError(t, obj.Remove(ctx), "successfully updated object with unknown-sized source but failed to remove") + } + // if err != nil: it's okay as long as no panic + }) + + }) + + // TestFsRootCollapse tests if the root of an fs "collapses" to the + // absolute root. It creates a new fs of the same backend type with its + // root set to a *nonexistent* folder, and attempts to read the info of + // an object in that folder, whose name is taken from a directory that + // exists in the absolute root. + // This test is added after + // https://github.com/rclone/rclone/issues/3164. + t.Run("FsRootCollapse", func(t *testing.T) { + deepRemoteName := subRemoteName + "/deeper/nonexisting/directory" + deepRemote, err := fs.NewFs(context.Background(), deepRemoteName) + require.NoError(t, err) + + colonIndex := strings.IndexRune(deepRemoteName, ':') + firstSlashIndex := strings.IndexRune(deepRemoteName, '/') + firstDir := deepRemoteName[colonIndex+1 : firstSlashIndex] + _, err = deepRemote.NewObject(ctx, firstDir) + require.Equal(t, fs.ErrorObjectNotFound, err) + // If err is not fs.ErrorObjectNotFound, it means the backend is + // somehow confused about root and absolute root. + }) + + // FsDirSetModTime tests setting the mod time on a directory if possible + t.Run("FsDirSetModTime", func(t *testing.T) { + const name = "dir-mod-time" + do := f.Features().DirSetModTime + if do == nil { + t.Skip("FS has no DirSetModTime interface") + } + + // Set ModTime on non existing directory should return error + t1 := fstest.Time("2001-02-03T04:05:06.499999999Z") + err := do(ctx, name, t1) + require.Error(t, err) + + // Make the directory and try again + err = f.Mkdir(ctx, name) + require.NoError(t, err) + err = do(ctx, name, t1) + require.NoError(t, err) + + // Check the modtime got set properly + dir := fstest.NewDirectory(ctx, t, f, name) + fstest.CheckDirModTime(ctx, t, f, dir, t1) + + // Tidy up + err = f.Rmdir(ctx, name) + require.NoError(t, err) + }) + + var testMetadata = fs.Metadata{ + // System metadata supported by all backends + "mtime": "2001-02-03T04:05:06.499999999Z", + // User metadata + "potato": "jersey", + } + var testMetadata2 = fs.Metadata{ + // System metadata supported by all backends + "mtime": "2002-02-03T04:05:06.499999999Z", + // User metadata + "potato": "king edwards", + } + + // FsMkdirMetadata tests creating a directory with metadata if possible + t.Run("FsMkdirMetadata", func(t *testing.T) { + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + const name = "dir-metadata" + do := f.Features().MkdirMetadata + if do == nil { + t.Skip("FS has no MkdirMetadata interface") + } + assert.True(t, f.Features().WriteDirMetadata, "Backends must support Directory.SetMetadata and Fs.MkdirMetadata") + + // Create the directory from fresh + dir, err := do(ctx, name, testMetadata) + require.NoError(t, err) + require.NotNil(t, dir) + + // Check the returned directory and one read from the listing + fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata) + fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata) + + // Now update the metadata on the existing directory + t.Run("Update", func(t *testing.T) { + dir, err := do(ctx, name, testMetadata2) + require.NoError(t, err) + require.NotNil(t, dir) + + // Check the returned directory and one read from the listing + fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata2) + // The TestUnionPolicy2 has randomness in it so it sets metadata on + // one directory but can read a different one from the listing. + if f.Name() != "TestUnionPolicy2" { + fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata2) + } + }) + + // Now test the Directory methods + t.Run("CheckDirectory", func(t *testing.T) { + _, ok := dir.(fs.Object) + assert.False(t, ok, "Directory must not type assert to Object") + _, ok = dir.(fs.ObjectInfo) + assert.False(t, ok, "Directory must not type assert to ObjectInfo") + }) + + // Tidy up + err = f.Rmdir(ctx, name) + require.NoError(t, err) + }) + + // FsDirectory checks methods on the directory object + t.Run("FsDirectory", func(t *testing.T) { + ctx, ci := fs.AddConfig(ctx) + ci.Metadata = true + const name = "dir-methods" + features := f.Features() + + if !features.CanHaveEmptyDirectories { + t.Skip("Can't test if can't have empty directories") + } + if !features.ReadDirMetadata && + !features.WriteDirMetadata && + !features.WriteDirSetModTime && + !features.UserDirMetadata && + !features.Overlay && + features.UnWrap == nil { + t.Skip("FS has no Directory methods and doesn't Wrap") + } + + // Create a directory to start with + err := f.Mkdir(ctx, name) + require.NoError(t, err) + + // Get the directory object + dir := fstest.NewDirectory(ctx, t, f, name) + _, ok := dir.(fs.Object) + assert.False(t, ok, "Directory must not type assert to Object") + _, ok = dir.(fs.ObjectInfo) + assert.False(t, ok, "Directory must not type assert to ObjectInfo") + + // Now test the directory methods + t.Run("ReadDirMetadata", func(t *testing.T) { + if !features.ReadDirMetadata { + t.Skip("Directories don't support ReadDirMetadata") + } + if f.Name() == "TestUnionPolicy3" { + t.Skipf("Test unreliable on %q", f.Name()) + } + fstest.CheckEntryMetadata(ctx, t, f, dir, fs.Metadata{ + "mtime": dir.ModTime(ctx).Format(time.RFC3339Nano), + }) + }) + + t.Run("WriteDirMetadata", func(t *testing.T) { + if !features.WriteDirMetadata { + t.Skip("Directories don't support WriteDirMetadata") + } + assert.NotNil(t, features.MkdirMetadata, "Backends must support Directory.SetMetadata and Fs.MkdirMetadata") + do, ok := dir.(fs.SetMetadataer) + require.True(t, ok, "Expected to find SetMetadata method on Directory") + err := do.SetMetadata(ctx, testMetadata) + require.NoError(t, err) + + fstest.CheckEntryMetadata(ctx, t, f, dir, testMetadata) + fstest.CheckEntryMetadata(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), testMetadata) + }) + + t.Run("WriteDirSetModTime", func(t *testing.T) { + if !features.WriteDirSetModTime { + t.Skip("Directories don't support WriteDirSetModTime") + } + assert.NotNil(t, features.DirSetModTime, "Backends must support Directory.SetModTime and Fs.DirSetModTime") + + t1 := fstest.Time("2001-02-03T04:05:10.123123123Z") + + do, ok := dir.(fs.SetModTimer) + require.True(t, ok, "Expected to find SetMetadata method on Directory") + err := do.SetModTime(ctx, t1) + require.NoError(t, err) + + fstest.CheckDirModTime(ctx, t, f, dir, t1) + fstest.CheckDirModTime(ctx, t, f, fstest.NewDirectory(ctx, t, f, name), t1) + }) + + // Check to see if Fs that wrap other Directories implement all the optional methods + t.Run("DirectoryCheckWrap", func(t *testing.T) { + if opt.SkipDirectoryCheckWrap { + t.Skip("Skipping DirectoryCheckWrap on this Fs") + } + if !features.Overlay && features.UnWrap == nil { + t.Skip("Not a wrapping Fs") + } + _, unsupported := fs.DirectoryOptionalInterfaces(dir) + for _, name := range unsupported { + if !stringsContains(name, opt.UnimplementableDirectoryMethods) { + t.Errorf("Missing Directory wrapper for %s", name) + } + } + }) + + // Tidy up + err = f.Rmdir(ctx, name) + require.NoError(t, err) + }) + + // Purge the folder + err = operations.Purge(ctx, f, "") + if !errors.Is(err, fs.ErrorDirNotFound) { + require.NoError(t, err) + } + purged = true + fstest.CheckListing(t, f, []fstest.Item{}) + + // Check purging again if not bucket-based + if !isBucketBasedButNotRoot(f) { + err = operations.Purge(ctx, f, "") + assert.Error(t, err, "Expecting error after on second purge") + if !errors.Is(err, fs.ErrorDirNotFound) { + t.Log("Warning: this should produce fs.ErrorDirNotFound") + } + } + + }) + + // Check directory is purged + if !purged { + _ = operations.Purge(ctx, f, "") + } + + t.Run("FsShutdown", func(t *testing.T) { + do := f.Features().Shutdown + if do == nil { + t.Skip("Shutdown method not supported") + } + require.NoError(t, do(ctx)) + require.NoError(t, do(ctx), "must be able to call Shutdown twice") + }) + + // Remove the local directory so we don't clutter up /tmp + if strings.HasPrefix(remoteName, "/") { + t.Log("remoteName", remoteName) + // Remove temp directory + err := os.Remove(remoteName) + require.NoError(t, err) + } +} diff --git a/fstest/license b/fstest/license new file mode 100644 index 0000000..bc452e3 --- /dev/null +++ b/fstest/license @@ -0,0 +1,19 @@ +Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/fstest/mockdir/dir.go b/fstest/mockdir/dir.go new file mode 100644 index 0000000..1036929 --- /dev/null +++ b/fstest/mockdir/dir.go @@ -0,0 +1,13 @@ +// Package mockdir makes a mock fs.Directory object +package mockdir + +import ( + "time" + + "github.com/rclone/rclone/fs" +) + +// New makes a mock directory object with the name given +func New(name string) fs.Directory { + return fs.NewDir(name, time.Time{}) +} diff --git a/fstest/mockfs/mockfs.go b/fstest/mockfs/mockfs.go new file mode 100644 index 0000000..68df1f6 --- /dev/null +++ b/fstest/mockfs/mockfs.go @@ -0,0 +1,153 @@ +// Package mockfs provides mock Fs for testing. +package mockfs + +import ( + "context" + "errors" + "fmt" + "io" + "path" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/hash" +) + +// Register with Fs +func Register() { + fs.Register(&fs.RegInfo{ + Name: "mockfs", + Description: "Mock FS", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "potato", + Help: "Does it have a potato?.", + Required: true, + }}, + }) +} + +// Fs is a minimal mock Fs +type Fs struct { + name string // the name of the remote + root string // The root directory (OS path) + features *fs.Features // optional features + rootDir fs.DirEntries // directory listing of root + hashes hash.Set // which hashes we support +} + +// ErrNotImplemented is returned by unimplemented methods +var ErrNotImplemented = errors.New("not implemented") + +// NewFs returns a new mock Fs +func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) { + f := &Fs{ + name: name, + root: root, + } + f.features = (&fs.Features{}).Fill(ctx, f) + return f, nil +} + +// AddObject adds an Object for List to return +// Only works for the root for the moment +func (f *Fs) AddObject(o fs.Object) { + f.rootDir = append(f.rootDir, o) + // Make this object part of mockfs if possible + do, ok := o.(interface{ SetFs(f fs.Fs) }) + if ok { + do.SetFs(f) + } +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String returns a description of the FS +func (f *Fs) String() string { + return fmt.Sprintf("Mock file system at %s", f.root) +} + +// Precision of the ModTimes in this Fs +func (f *Fs) Precision() time.Duration { + return time.Second +} + +// Hashes returns the supported hash types of the filesystem +func (f *Fs) Hashes() hash.Set { + return f.hashes +} + +// SetHashes sets the hashes that this supports +func (f *Fs) SetHashes(hashes hash.Set) { + f.hashes = hashes +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + if dir == "" { + return f.rootDir, nil + } + return entries, fs.ErrorDirNotFound +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error ErrorObjectNotFound. +func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + dirPath := path.Dir(remote) + if dirPath == "" || dirPath == "." { + for _, entry := range f.rootDir { + if entry.Remote() == remote { + return entry.(fs.Object), nil + } + } + } + return nil, fs.ErrorObjectNotFound +} + +// Put in to the remote path with the modTime given of the given size +// +// May create the object even if it returns an error - if so +// will return the object and the error, otherwise will return +// nil and the error +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return nil, ErrNotImplemented +} + +// Mkdir makes the directory (container, bucket) +// +// Shouldn't return an error if it already exists +func (f *Fs) Mkdir(ctx context.Context, dir string) error { + return ErrNotImplemented +} + +// Rmdir removes the directory (container, bucket) if empty +// +// Return an error if it doesn't exist or isn't empty +func (f *Fs) Rmdir(ctx context.Context, dir string) error { + return ErrNotImplemented +} + +// Assert it is the correct type +var _ fs.Fs = (*Fs)(nil) diff --git a/fstest/mockobject/mockobject.go b/fstest/mockobject/mockobject.go new file mode 100644 index 0000000..89f0fab --- /dev/null +++ b/fstest/mockobject/mockobject.go @@ -0,0 +1,235 @@ +// Package mockobject provides a mock object which can be created from a string +package mockobject + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/hash" +) + +var errNotImpl = errors.New("not implemented") + +// Object is a mock fs.Object useful for testing +type Object string + +// New returns mock fs.Object useful for testing +func New(name string) Object { + return Object(name) +} + +// String returns a description of the Object +func (o Object) String() string { + return string(o) +} + +// Fs returns read only access to the Fs that this object is part of +func (o Object) Fs() fs.Info { + return nil +} + +// Remote returns the remote path +func (o Object) Remote() string { + return string(o) +} + +// Hash returns the selected checksum of the file +// If no checksum is available it returns "" +func (o Object) Hash(ctx context.Context, t hash.Type) (string, error) { + return "", errNotImpl +} + +// ModTime returns the modification date of the file +// It should return a best guess if one isn't available +func (o Object) ModTime(ctx context.Context) (t time.Time) { + return t +} + +// Size returns the size of the file +func (o Object) Size() int64 { return 0 } + +// Storable says whether this object can be stored +func (o Object) Storable() bool { + return true +} + +// SetModTime sets the metadata on the object to set the modification date +func (o Object) SetModTime(ctx context.Context, t time.Time) error { + return errNotImpl +} + +// Open opens the file for read. Call Close() on the returned io.ReadCloser +func (o Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { + return nil, errNotImpl +} + +// Update in to the object with the modTime given of the given size +func (o Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + return errNotImpl +} + +// Remove this object +func (o Object) Remove(ctx context.Context) error { + return errNotImpl +} + +// SeekMode specifies the optional Seek interface for the ReadCloser returned by Open +type SeekMode int + +const ( + // SeekModeNone specifies no seek interface + SeekModeNone SeekMode = iota + // SeekModeRegular specifies the regular io.Seek interface + SeekModeRegular + // SeekModeRange specifies the fs.RangeSeek interface + SeekModeRange +) + +// SeekModes contains all valid SeekMode's +var SeekModes = []SeekMode{SeekModeNone, SeekModeRegular, SeekModeRange} + +// ContentMockObject mocks an fs.Object and has content, mod time +type ContentMockObject struct { + Object + content []byte + seekMode SeekMode + f fs.Fs + unknownSize bool + modTime time.Time +} + +// WithContent returns an fs.Object with the given content. +func (o Object) WithContent(content []byte, mode SeekMode) *ContentMockObject { + return &ContentMockObject{ + Object: o, + content: content, + seekMode: mode, + } +} + +// SetFs sets the return value of the Fs() call +func (o *ContentMockObject) SetFs(f fs.Fs) { + o.f = f +} + +// SetUnknownSize makes the mock object return -1 for size if true +func (o *ContentMockObject) SetUnknownSize(unknownSize bool) { + o.unknownSize = unknownSize +} + +// Fs returns read only access to the Fs that this object is part of +// +// This is nil unless SetFs has been called +func (o *ContentMockObject) Fs() fs.Info { + return o.f +} + +// Open opens the file for read. Call Close() on the returned io.ReadCloser +func (o *ContentMockObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { + size := int64(len(o.content)) + var offset, limit int64 = 0, -1 + for _, option := range options { + switch x := option.(type) { + case *fs.SeekOption: + offset = x.Offset + case *fs.RangeOption: + offset, limit = x.Decode(size) + default: + if option.Mandatory() { + return nil, fmt.Errorf("unsupported mandatory option: %v", option) + } + } + } + if limit == -1 || offset+limit > size { + limit = size - offset + } + + var r *bytes.Reader + if o.seekMode == SeekModeNone { + r = bytes.NewReader(o.content[offset : offset+limit]) + } else { + r = bytes.NewReader(o.content) + _, err := r.Seek(offset, io.SeekStart) + if err != nil { + return nil, err + } + } + switch o.seekMode { + case SeekModeNone: + return &readCloser{r}, nil + case SeekModeRegular: + return &readSeekCloser{r}, nil + case SeekModeRange: + return &readRangeSeekCloser{r}, nil + default: + return nil, errors.New(o.seekMode.String()) + } +} + +// Size returns the size of the file +func (o *ContentMockObject) Size() int64 { + if o.unknownSize { + return -1 + } + return int64(len(o.content)) +} + +// Hash returns the selected checksum of the file +// If no checksum is available it returns "" +func (o *ContentMockObject) Hash(ctx context.Context, t hash.Type) (string, error) { + hasher, err := hash.NewMultiHasherTypes(hash.NewHashSet(t)) + if err != nil { + return "", err + } + _, err = hasher.Write(o.content) + if err != nil { + return "", err + } + return hasher.Sums()[t], nil +} + +// ModTime returns the modification date of the file +// It should return a best guess if one isn't available +func (o *ContentMockObject) ModTime(ctx context.Context) time.Time { + return o.modTime +} + +// SetModTime sets the metadata on the object to set the modification date +func (o *ContentMockObject) SetModTime(ctx context.Context, t time.Time) error { + o.modTime = t + return nil +} + +type readCloser struct{ io.Reader } + +func (r *readCloser) Close() error { return nil } + +type readSeekCloser struct{ io.ReadSeeker } + +func (r *readSeekCloser) Close() error { return nil } + +type readRangeSeekCloser struct{ io.ReadSeeker } + +func (r *readRangeSeekCloser) RangeSeek(offset int64, whence int, length int64) (int64, error) { + return r.ReadSeeker.Seek(offset, whence) +} + +func (r *readRangeSeekCloser) Close() error { return nil } + +func (m SeekMode) String() string { + switch m { + case SeekModeNone: + return "SeekModeNone" + case SeekModeRegular: + return "SeekModeRegular" + case SeekModeRange: + return "SeekModeRange" + default: + return fmt.Sprintf("SeekModeInvalid(%d)", m) + } +} diff --git a/fstest/run.go b/fstest/run.go new file mode 100644 index 0000000..2f492d4 --- /dev/null +++ b/fstest/run.go @@ -0,0 +1,398 @@ +/* + +This provides Run for use in creating test suites + +To use this declare a TestMain + +// TestMain drives the tests +func TestMain(m *testing.M) { + fstest.TestMain(m) +} + +And then make and destroy a Run in each test + +func TestMkdir(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + // test stuff +} + +This will make r.Fremote and r.Flocal for a remote and a local +remote. The remote is determined by the -remote flag passed in. + +*/ + +package fstest + +import ( + "bytes" + "context" + "flag" + "fmt" + "log" + "os" + "path" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/object" + "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/lib/file" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Run holds the remotes for a test run +type Run struct { + LocalName string + Flocal fs.Fs + Fremote fs.Fs + FremoteName string + Precision time.Duration + cleanRemote func() + mkdir map[string]bool // whether the remote has been made yet for the fs name + Logf, Fatalf func(text string, args ...any) +} + +// ResetRun re-reads the command line arguments into the global run. +func ResetRun() { + oneRun = newRun() +} + +// TestMain drives the tests +func TestMain(m *testing.M) { + flag.Parse() + if !*Individual { + ResetRun() + } + rc := m.Run() + if !*Individual { + oneRun.Finalise() + } + os.Exit(rc) +} + +// oneRun holds the master run data if individual is not set +var oneRun *Run + +// newRun initialise the remote and local for testing and returns a +// run object. +// +// r.Flocal is an empty local Fs +// r.Fremote is an empty remote Fs +// +// Finalise() will tidy them away when done. +func newRun() *Run { + r := &Run{ + Logf: log.Printf, + Fatalf: log.Fatalf, + mkdir: make(map[string]bool), + } + + Initialise() + + var err error + r.Fremote, r.FremoteName, r.cleanRemote, err = RandomRemote() + if err != nil { + r.Fatalf("Failed to open remote %q: %v", *RemoteName, err) + } + + r.LocalName, err = os.MkdirTemp("", "rclone") + if err != nil { + r.Fatalf("Failed to create temp dir: %v", err) + } + r.LocalName = filepath.ToSlash(r.LocalName) + r.Flocal, err = fs.NewFs(context.Background(), r.LocalName) + if err != nil { + r.Fatalf("Failed to make %q: %v", r.LocalName, err) + } + + r.Precision = fs.GetModifyWindow(context.Background(), r.Fremote, r.Flocal) + + return r +} + +// run f(), retrying it until it returns with no error or the limit +// expires and it calls t.Fatalf +func retry(t *testing.T, what string, f func() error) { + var err error + for try := 1; try <= *ListRetries; try++ { + err = f() + if err == nil { + return + } + t.Logf("%s failed - try %d/%d: %v", what, try, *ListRetries, err) + time.Sleep(time.Second) + } + t.Logf("%s failed: %v", what, err) +} + +// newRunIndividual initialise the remote and local for testing and +// returns a run object. Pass in true to make individual tests or +// false to use the global one. +// +// r.Flocal is an empty local Fs +// r.Fremote is an empty remote Fs +// +// Finalise() will tidy them away when done. +func newRunIndividual(t *testing.T, individual bool) *Run { + ctx := context.Background() + var r *Run + if individual { + r = newRun() + } else { + // If not individual, use the global one with the clean method overridden + r = new(Run) + *r = *oneRun + r.cleanRemote = func() { + var toDelete []string + err := walk.ListR(ctx, r.Fremote, "", true, -1, walk.ListAll, func(entries fs.DirEntries) error { + for _, entry := range entries { + switch x := entry.(type) { + case fs.Object: + retry(t, fmt.Sprintf("removing file %q", x.Remote()), func() error { return x.Remove(ctx) }) + case fs.Directory: + toDelete = append(toDelete, x.Remote()) + } + } + return nil + }) + if err == fs.ErrorDirNotFound { + return + } + require.NoError(t, err) + sort.Strings(toDelete) + for i := len(toDelete) - 1; i >= 0; i-- { + dir := toDelete[i] + retry(t, fmt.Sprintf("removing dir %q", dir), func() error { + return r.Fremote.Rmdir(ctx, dir) + }) + } + // Check remote is empty + CheckListingWithPrecision(t, r.Fremote, []Item{}, []string{}, r.Fremote.Precision()) + // Clear the remote cache + cache.Clear() + } + } + r.Logf = t.Logf + r.Fatalf = t.Fatalf + r.Logf("Remote %q, Local %q, Modify Window %q", r.Fremote, r.Flocal, fs.GetModifyWindow(ctx, r.Fremote)) + t.Cleanup(r.Finalise) + return r +} + +// NewRun initialise the remote and local for testing and returns a +// run object. Call this from the tests. +// +// r.Flocal is an empty local Fs +// r.Fremote is an empty remote Fs +func NewRun(t *testing.T) *Run { + return newRunIndividual(t, *Individual) +} + +// NewRunIndividual as per NewRun but makes an individual remote for this test +func NewRunIndividual(t *testing.T) *Run { + return newRunIndividual(t, true) +} + +// RenameFile renames a file in local +func (r *Run) RenameFile(item Item, newpath string) Item { + oldFilepath := path.Join(r.LocalName, item.Path) + newFilepath := path.Join(r.LocalName, newpath) + if err := os.Rename(oldFilepath, newFilepath); err != nil { + r.Fatalf("Failed to rename file from %q to %q: %v", item.Path, newpath, err) + } + + item.Path = newpath + + return item +} + +// WriteFile writes a file to local +func (r *Run) WriteFile(filePath, content string, t time.Time) Item { + item := NewItem(filePath, content, t) + // FIXME make directories? + filePath = path.Join(r.LocalName, filePath) + dirPath := path.Dir(filePath) + err := file.MkdirAll(dirPath, 0770) + if err != nil { + r.Fatalf("Failed to make directories %q: %v", dirPath, err) + } + err = os.WriteFile(filePath, []byte(content), 0600) + if err != nil { + r.Fatalf("Failed to write file %q: %v", filePath, err) + } + err = os.Chtimes(filePath, t, t) + if err != nil { + r.Fatalf("Failed to chtimes file %q: %v", filePath, err) + } + return item +} + +// ForceMkdir creates the remote +func (r *Run) ForceMkdir(ctx context.Context, f fs.Fs) { + err := f.Mkdir(ctx, "") + if err != nil { + r.Fatalf("Failed to mkdir %q: %v", f, err) + } + r.mkdir[f.String()] = true +} + +// Mkdir creates the remote if it hasn't been created already +func (r *Run) Mkdir(ctx context.Context, f fs.Fs) { + if !r.mkdir[f.String()] { + r.ForceMkdir(ctx, f) + } +} + +// WriteObjectTo writes an object to the fs, remote passed in +func (r *Run) WriteObjectTo(ctx context.Context, f fs.Fs, remote, content string, modTime time.Time, useUnchecked bool) Item { + put := f.Put + if useUnchecked { + put = f.Features().PutUnchecked + if put == nil { + r.Fatalf("Fs doesn't support PutUnchecked") + } + } + r.Mkdir(ctx, f) + + // calculate all hashes f supports for content + hash, err := hash.NewMultiHasherTypes(f.Hashes()) + if err != nil { + r.Fatalf("Failed to make new multi hasher: %v", err) + } + _, err = hash.Write([]byte(content)) + if err != nil { + r.Fatalf("Failed to make write to hash: %v", err) + } + hashSums := hash.Sums() + + const maxTries = 10 + for tries := 1; ; tries++ { + in := bytes.NewBufferString(content) + objinfo := object.NewStaticObjectInfo(remote, modTime, int64(len(content)), true, hashSums, nil) + _, err := put(ctx, in, objinfo) + if err == nil { + break + } + // Retry if err returned a retry error + if fserrors.IsRetryError(err) && tries < maxTries { + r.Logf("Retry Put of %q to %v: %d/%d (%v)", remote, f, tries, maxTries, err) + time.Sleep(2 * time.Second) + continue + } + r.Fatalf("Failed to put %q to %q: %v", remote, f, err) + } + return NewItem(remote, content, modTime) +} + +// WriteObject writes an object to the remote +func (r *Run) WriteObject(ctx context.Context, remote, content string, modTime time.Time) Item { + return r.WriteObjectTo(ctx, r.Fremote, remote, content, modTime, false) +} + +// WriteUncheckedObject writes an object to the remote not checking for duplicates +func (r *Run) WriteUncheckedObject(ctx context.Context, remote, content string, modTime time.Time) Item { + return r.WriteObjectTo(ctx, r.Fremote, remote, content, modTime, true) +} + +// WriteBoth calls WriteObject and WriteFile with the same arguments +func (r *Run) WriteBoth(ctx context.Context, remote, content string, modTime time.Time) Item { + r.WriteFile(remote, content, modTime) + return r.WriteObject(ctx, remote, content, modTime) +} + +// CheckWithDuplicates does a test but allows duplicates +func (r *Run) CheckWithDuplicates(t *testing.T, items ...Item) { + var want, got []string + + // construct a []string of desired items + for _, item := range items { + want = append(want, fmt.Sprintf("%q %d", item.Path, item.Size)) + } + sort.Strings(want) + + // do the listing + objs, _, err := walk.GetAll(context.Background(), r.Fremote, "", true, -1) + if err != nil && err != fs.ErrorDirNotFound { + t.Fatalf("Error listing: %v", err) + } + + // construct a []string of actual items + for _, o := range objs { + got = append(got, fmt.Sprintf("%q %d", o.Remote(), o.Size())) + } + sort.Strings(got) + + assert.Equal(t, want, got) +} + +// CheckLocalItems checks the local fs with proper precision +// to see if it has the expected items. +func (r *Run) CheckLocalItems(t *testing.T, items ...Item) { + CheckItemsWithPrecision(t, r.Flocal, r.Precision, items...) +} + +// CheckRemoteItems checks the remote fs with proper precision +// to see if it has the expected items. +func (r *Run) CheckRemoteItems(t *testing.T, items ...Item) { + CheckItemsWithPrecision(t, r.Fremote, r.Precision, items...) +} + +// CheckLocalListing checks the local fs with proper precision +// to see if it has the expected contents. +// +// If expectedDirs is non nil then we check those too. Note that no +// directories returned is also OK as some remotes don't return +// directories. +func (r *Run) CheckLocalListing(t *testing.T, items []Item, expectedDirs []string) { + CheckListingWithPrecision(t, r.Flocal, items, expectedDirs, r.Precision) +} + +// CheckRemoteListing checks the remote fs with proper precision +// to see if it has the expected contents. +// +// If expectedDirs is non nil then we check those too. Note that no +// directories returned is also OK as some remotes don't return +// directories. +func (r *Run) CheckRemoteListing(t *testing.T, items []Item, expectedDirs []string) { + CheckListingWithPrecision(t, r.Fremote, items, expectedDirs, r.Precision) +} + +// CheckDirectoryModTimes checks that the directory names in r.Flocal has the correct modtime compared to r.Fremote +func (r *Run) CheckDirectoryModTimes(t *testing.T, names ...string) { + if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil { + fs.Debugf(r.Fremote, "Skipping modtime test as remote does not support DirSetModTime or MkdirMetadata") + return + } + ctx := context.Background() + for _, name := range names { + wantT := NewDirectory(ctx, t, r.Flocal, name).ModTime(ctx) + got := NewDirectory(ctx, t, r.Fremote, name) + CheckDirModTime(ctx, t, r.Fremote, got, wantT) + } +} + +// Clean the temporary directory +func (r *Run) cleanTempDir() { + err := os.RemoveAll(r.LocalName) + if err != nil { + r.Logf("Failed to clean temporary directory %q: %v", r.LocalName, err) + } +} + +// Finalise cleans the remote and local +func (r *Run) Finalise() { + // r.Logf("Cleaning remote %q", r.Fremote) + r.cleanRemote() + // r.Logf("Cleaning local %q", r.LocalName) + r.cleanTempDir() + // Clear the remote cache + cache.Clear() +} diff --git a/fstest/runs/config.go b/fstest/runs/config.go new file mode 100644 index 0000000..82b19c8 --- /dev/null +++ b/fstest/runs/config.go @@ -0,0 +1,195 @@ +// Config handling + +// Package runs provides the types used by test_all +package runs + +import ( + "fmt" + "os" + "path" + "slices" + + "github.com/rclone/rclone/fs" + yaml "gopkg.in/yaml.v3" +) + +// Test describes an integration test to run with `go test` +type Test struct { + Path string // path to the source directory + FastList bool // if it is possible to add -fast-list to tests + Short bool // if it is possible to run the test with -short + AddBackend bool // set if Path needs the current backend appending + NoRetries bool // set if no retries should be performed + NoBinary bool // set to not build a binary in advance + LocalOnly bool // if set only run with the local backend +} + +// Backend describes a backend test +// +// FIXME make bucket-based remotes set sub-dir automatically??? +type Backend struct { + Backend string // name of the backend directory + Remote string // name of the test remote + FastList bool // set to test with -fast-list + Short bool // set to test with -short + OneOnly bool // set to run only one backend test at once + MaxFile string // file size limit + CleanUp bool // when running clean, run cleanup first + Ignore []string // test names to ignore the failure of + Tests []string // paths of tests to run, blank for all + IgnoreTests []string // paths of tests not to run, blank for none + ListRetries int // -list-retries if > 0 + ExtraTime float64 // factor to multiply the timeout by + Env []string // environment variables to set in form KEY=VALUE +} + +// includeTest returns true if this backend should be included in this +// test +func (b *Backend) includeTest(t *Test) bool { + // Is this test ignored + if slices.Contains(b.IgnoreTests, t.Path) { + return false + } + // Empty b.Tests imples do all of them except the ignored + if len(b.Tests) == 0 { + return true + } + return slices.Contains(b.Tests, t.Path) +} + +// MakeRuns creates Run objects the Backend and Test +// +// There can be several created, one for each combination of optional +// flags (e.g. FastList) +func (b *Backend) MakeRuns(t *Test) (runs []*Run) { + if !b.includeTest(t) { + return runs + } + maxSize := fs.SizeSuffix(0) + if b.MaxFile != "" { + if err := maxSize.Set(b.MaxFile); err != nil { + fs.Logf(nil, "Invalid maxfile value %q: %v", b.MaxFile, err) + } + } + fastlists := []bool{false} + if b.FastList && t.FastList { + fastlists = append(fastlists, true) + } + ignore := make(map[string]struct{}, len(b.Ignore)) + for _, item := range b.Ignore { + ignore[item] = struct{}{} + } + for _, fastlist := range fastlists { + if t.LocalOnly && b.Backend != "local" { + continue + } + run := &Run{ + Remote: b.Remote, + Backend: b.Backend, + Path: t.Path, + FastList: fastlist, + Short: (b.Short && t.Short), + NoRetries: t.NoRetries, + OneOnly: b.OneOnly, + NoBinary: t.NoBinary, + SizeLimit: int64(maxSize), + Ignore: ignore, + ListRetries: b.ListRetries, + ExtraTime: b.ExtraTime, + Env: b.Env, + } + if t.AddBackend { + run.Path = path.Join(run.Path, b.Backend) + } + runs = append(runs, run) + } + return runs +} + +// Config describes the config for this program +type Config struct { + Tests []Test + Backends []Backend +} + +// NewConfig reads the config file +func NewConfig(configFile string) (*Config, error) { + d, err := os.ReadFile(configFile) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + config := &Config{} + err = yaml.Unmarshal(d, &config) + if err != nil { + return nil, fmt.Errorf("failed to parse config file: %w", err) + } + // d, err = yaml.Marshal(&config) + // if err != nil { + // log.Fatalf("error: %v", err) + // } + // fmt.Printf("--- m dump:\n%s\n\n", string(d)) + return config, nil +} + +// MakeRuns makes Run objects for each combination of Backend and Test +// in the config +func (c *Config) MakeRuns() (runs Runs) { + for _, backend := range c.Backends { + for _, test := range c.Tests { + runs = append(runs, backend.MakeRuns(&test)...) + } + } + return runs +} + +// FilterBackendsByRemotes filters the Backends with the remotes passed in. +// +// If no backend is found with a remote is found then synthesize one +func (c *Config) FilterBackendsByRemotes(remotes []string) { + var newBackends []Backend + for _, name := range remotes { + found := false + for i := range c.Backends { + if c.Backends[i].Remote == name { + newBackends = append(newBackends, c.Backends[i]) + found = true + } + } + if !found { + fs.Logf(nil, "Remote %q not found - inserting with default flags", name) + // Lookup which backend + fsInfo, _, _, _, err := fs.ConfigFs(name) + if err != nil { + fs.Fatalf(nil, "couldn't find remote %q: %v", name, err) + } + newBackends = append(newBackends, Backend{Backend: fsInfo.FileName(), Remote: name}) + } + } + c.Backends = newBackends +} + +// FilterBackendsByBackends filters the Backends with the backendNames passed in +func (c *Config) FilterBackendsByBackends(backendNames []string) { + var newBackends []Backend + for _, name := range backendNames { + for i := range c.Backends { + if c.Backends[i].Backend == name { + newBackends = append(newBackends, c.Backends[i]) + } + } + } + c.Backends = newBackends +} + +// FilterTests filters the incoming tests into the backends selected +func (c *Config) FilterTests(paths []string) { + var newTests []Test + for _, path := range paths { + for i := range c.Tests { + if c.Tests[i].Path == path { + newTests = append(newTests, c.Tests[i]) + } + } + } + c.Tests = newTests +} diff --git a/fstest/runs/report.go b/fstest/runs/report.go new file mode 100644 index 0000000..39787a2 --- /dev/null +++ b/fstest/runs/report.go @@ -0,0 +1,333 @@ +package runs + +import ( + "bytes" + "encoding/json" + "fmt" + "html/template" + "os" + "os/exec" + "path" + "runtime" + "sort" + "strings" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/lib/file" + "github.com/skratchdot/open-golang/open" +) + +const timeFormat = "2006-01-02-150405" + +// Report holds the info to make a report on a series of test runs +type Report struct { + LogDir string // output directory for logs and report + StartTime time.Time // time started + DateTime string // directory name for output + Duration time.Duration // time the run took + Failed Runs // failed runs + Passed Runs // passed runs + Runs []ReportRun // runs to report + Version string // rclone version + Previous string // previous test name if known + IndexHTML string // path to the index.html file + URL string // online version + Branch string // rclone branch + Commit string // rclone commit + GOOS string // Go OS + GOARCH string // Go Arch + GoVersion string // Go Version +} + +// ReportRun is used in the templates to report on a test run +type ReportRun struct { + Name string + Runs Runs +} + +// FIXME take -issue or -pr parameter... + +// NewReport initialises and returns a Report +func NewReport(Opt RunOpt) *Report { + r := &Report{ + StartTime: time.Now(), + Version: fs.Version, + GOOS: runtime.GOOS, + GOARCH: runtime.GOARCH, + GoVersion: runtime.Version(), + } + r.DateTime = r.StartTime.Format(timeFormat) + + // Find previous log directory if possible + names, err := os.ReadDir(Opt.OutputDir) + if err == nil && len(names) > 0 { + r.Previous = names[len(names)-1].Name() + } + + // Create output directory for logs and report + r.LogDir = path.Join(Opt.OutputDir, r.DateTime) + err = file.MkdirAll(r.LogDir, 0777) + if err != nil { + fs.Fatalf(nil, "Failed to make log directory: %v", err) + } + + // Online version + r.URL = Opt.URLBase + r.DateTime + "/index.html" + + // Get branch/commit + r.Branch, r.Commit = gitBranchAndCommit() + + return r +} + +// gitBranchAndCommit returns the current branch and commit hash. +// +// It returns "" on error. +func gitBranchAndCommit() (branch, commit string) { + // branch (empty if detached) + var b bytes.Buffer + cmdB := exec.Command("git", "symbolic-ref", "--short", "-q", "HEAD") + cmdB.Stdout = &b + if e := cmdB.Run(); e == nil { + branch = strings.TrimSpace(b.String()) + } + + // commit (full SHA) + var c bytes.Buffer + cmdC := exec.Command("git", "rev-parse", "HEAD") + cmdC.Stdout = &c + if e := cmdC.Run(); e == nil { + commit = strings.TrimSpace(c.String()) + } + + return branch, commit +} + +// End should be called when the tests are complete +func (r *Report) End() { + r.Duration = time.Since(r.StartTime) + sort.Sort(r.Failed) + sort.Sort(r.Passed) + r.Runs = []ReportRun{ + {Name: "Failed", Runs: r.Failed}, + {Name: "Passed", Runs: r.Passed}, + } +} + +// AllPassed returns true if there were no failed tests +func (r *Report) AllPassed() bool { + return len(r.Failed) == 0 +} + +// RecordResult should be called with a Run when it has finished to be +// recorded into the Report +func (r *Report) RecordResult(t *Run) { + if !t.passed() { + r.Failed = append(r.Failed, t) + } else { + r.Passed = append(r.Passed, t) + } +} + +// Title returns a human-readable summary title for the Report +func (r *Report) Title() string { + if r.AllPassed() { + return fmt.Sprintf("PASS: All tests finished OK in %v", r.Duration) + } + return fmt.Sprintf("FAIL: %d tests failed in %v", len(r.Failed), r.Duration) +} + +// LogSummary writes the summary to the log file +func (r *Report) LogSummary() { + fs.Logf(nil, "Logs in %q", r.LogDir) + + // Summarise results + fs.Logf(nil, "SUMMARY") + fs.Log(nil, r.Title()) + if !r.AllPassed() { + for _, t := range r.Failed { + fs.Logf(nil, " * %s", toShell(t.nextCmdLine())) + fs.Logf(nil, " * Failed tests: %v", t.FailedTests) + } + } +} + +// LogJSON writes the summary to index.json in LogDir +func (r *Report) LogJSON() { + out, err := json.MarshalIndent(r, "", "\t") + if err != nil { + fs.Fatalf(nil, "Failed to marshal data for index.json: %v", err) + } + err = os.WriteFile(path.Join(r.LogDir, "index.json"), out, 0666) + if err != nil { + fs.Fatalf(nil, "Failed to write index.json: %v", err) + } +} + +// LogHTML writes the summary to index.html in LogDir +func (r *Report) LogHTML() { + r.IndexHTML = path.Join(r.LogDir, "index.html") + out, err := os.Create(r.IndexHTML) + if err != nil { + fs.Fatalf(nil, "Failed to open index.html: %v", err) + } + defer func() { + err := out.Close() + if err != nil { + fs.Fatalf(nil, "Failed to close index.html: %v", err) + } + }() + err = reportTemplate.Execute(out, r) + if err != nil { + fs.Fatalf(nil, "Failed to execute template: %v", err) + } + _ = open.Start("file://" + r.IndexHTML) +} + +var reportHTML = `<!DOCTYPE html> +<html lang="en"> +<head> +<meta charset="utf-8"> +<title>{{ .Title }}</title> +<style> +table { + border-collapse: collapse; + border-spacing: 0; + border: 1px solid #ddd; +} +table.tests { + width: 100%; +} +table, th, td { + border: 1px solid #264653; +} +.Failed { + color: #BE5B43; +} +.Passed { + color: #17564E; +} +.false { + font-weight: lighter; +} +.true { + font-weight: bold; +} + +th, td { + text-align: left; + padding: 4px; +} + +tr:nth-child(even) { + background-color: #f2f2f2; +} + +a { + color: #5B1955; + text-decoration: none; +} +a:hover, a:focus { + color: #F4A261; + text-decoration:underline; +} +a:focus { + outline: thin dotted; + outline: 5px auto; +} +</style> +</head> +<body> +<h1>{{ .Title }}</h1> + +<table> +<tr><th>Version</th><td>{{ .Version }}</td></tr> +<tr><th>Test</th><td><a href="{{ .URL }}">{{ .DateTime}}</a></td></tr> +<tr><th>Branch</th><td><a href="https://github.com/rclone/rclone/tree/{{ .Branch }}">{{ .Branch }}</a></td></tr> +{{ if .Commit}}<tr><th>Commit</th><td><a href="https://github.com/rclone/rclone/commit/{{ .Commit }}">{{ .Commit }}</a></td></tr>{{ end }} +<tr><th>Go</th><td>{{ .GoVersion }} {{ .GOOS }}/{{ .GOARCH }}</td></tr> +<tr><th>Duration</th><td>{{ .Duration }}</td></tr> +{{ if .Previous}}<tr><th>Previous</th><td><a href="../{{ .Previous }}/index.html">{{ .Previous }}</a></td></tr>{{ end }} +<tr><th>Up</th><td><a href="../">Older Tests</a></td></tr> +</table> + +{{ range .Runs }} +{{ if .Runs }} +<h2 class="{{ .Name }}">{{ .Name }}: {{ len .Runs }}</h2> +<table class="{{ .Name }} tests"> +<tr> +<th>Backend</th> +<th>Remote</th> +<th>Test</th> +<th>FastList</th> +<th>Failed</th> +<th>Logs</th> +</tr> +{{ $prevBackend := "" }} +{{ $prevRemote := "" }} +{{ range .Runs}} +<tr> +<td>{{ if ne $prevBackend .Backend }}{{ .Backend }}{{ end }}{{ $prevBackend = .Backend }}</td> +<td>{{ if ne $prevRemote .Remote }}{{ .Remote }}{{ end }}{{ $prevRemote = .Remote }}</td> +<td>{{ .Path }}</td> +<td><span class="{{ .FastList }}">{{ .FastList }}</span></td> +<td>{{ .FailedTestsCSV }}</td> +<td>{{ range $i, $v := .Logs }}<a href="{{ $v }}">#{{ $i }}</a> {{ end }}</td> +</tr> +{{ end }} +</table> +{{ end }} +{{ end }} +</body> +</html> +` + +var reportTemplate = template.Must(template.New("Report").Parse(reportHTML)) + +// EmailHTML sends the summary report to the email address supplied +func (r *Report) EmailHTML(Opt RunOpt) { + if Opt.EmailReport == "" || r.IndexHTML == "" { + return + } + fs.Logf(nil, "Sending email summary to %q", Opt.EmailReport) + cmdLine := []string{"mail", "-a", "Content-Type: text/html", Opt.EmailReport, "-s", "rclone integration tests: " + r.Title()} + cmd := exec.Command(cmdLine[0], cmdLine[1:]...) + in, err := os.Open(r.IndexHTML) + if err != nil { + fs.Fatalf(nil, "Failed to open index.html: %v", err) + } + cmd.Stdin = in + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + fs.Fatalf(nil, "Failed to send email: %v", err) + } + _ = in.Close() +} + +// uploadTo uploads a copy of the report online to the dir given +func (r *Report) uploadTo(Opt RunOpt, uploadDir string) { + dst := path.Join(Opt.UploadPath, uploadDir) + fs.Logf(nil, "Uploading results to %q", dst) + cmdLine := []string{"rclone", "sync", "--stats-log-level", "NOTICE", r.LogDir, dst} + cmd := exec.Command(cmdLine[0], cmdLine[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + fs.Fatalf(nil, "Failed to upload results: %v", err) + } +} + +// Upload uploads a copy of the report online +func (r *Report) Upload(Opt RunOpt) { + if Opt.UploadPath == "" || r.IndexHTML == "" { + return + } + // Upload into dated directory + r.uploadTo(Opt, r.DateTime) + // And again into current + r.uploadTo(Opt, "current") +} diff --git a/fstest/runs/run.go b/fstest/runs/run.go new file mode 100644 index 0000000..c898d39 --- /dev/null +++ b/fstest/runs/run.go @@ -0,0 +1,481 @@ +// Run a test + +package runs + +import ( + "bytes" + "context" + "fmt" + "go/build" + "io" + "os" + "os/exec" + "path" + "regexp" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fstest/testserver" +) + +// Control concurrency per backend if required +var ( + oneOnlyMu sync.Mutex + oneOnly = map[string]*sync.Mutex{} +) + +// RunOpt holds the options for the Run +type RunOpt struct { + MaxTries int // Number of times to try each test + MaxN int // Maximum number of tests to run at once + TestRemotes string // Comma separated list of remotes to test, e.g. 'TestSwift:,TestS3' + TestBackends string // Comma separated list of backends to test, e.g. 's3,googlecloudstorage + TestTests string // Comma separated list of tests to test, e.g. 'fs/sync,fs/operations' + Clean bool // Instead of testing, clean all left over test directories + RunOnly string // Run only those tests matching the regexp supplied + Timeout time.Duration // Maximum time to run each test for before giving up + Race bool // If set run the tests under the race detector + ConfigFile string // Path to config file + OutputDir string // Place to store results + EmailReport string // Set to email the report to the address supplied + DryRun bool // Print commands which would be executed only + URLBase string // Base for the online version + UploadPath string // Set this to an rclone path to upload the results here + Verbose bool // Set to enable verbose logging in the tests + ListRetries int // Number or times to retry listing - set to override the default +} + +// Run holds info about a running test +// +// A run just runs one command line, but it can be run multiple times +// if retries are needed. +type Run struct { + // Config + Remote string // name of the test remote + Backend string // name of the backend + Path string // path to the source directory + FastList bool // add -fast-list to tests + Short bool // add -short + NoRetries bool // don't retry if set + OneOnly bool // only run test for this backend at once + NoBinary bool // set to not build a binary + SizeLimit int64 // maximum test file size + Ignore map[string]struct{} + ListRetries int // -list-retries if > 0 + ExtraTime float64 // multiply the timeout by this + Env []string // environment variables in form KEY=VALUE + // Internals + CmdLine []string + CmdString string + Try int + err error + output []byte + FailedTests []string + RunFlag string + LogDir string // directory to place the logs + TrialName string // name/log file name of current trial + TrialNames []string // list of all the trials +} + +// Runs records multiple Run objects +type Runs []*Run + +// Sort interface +func (rs Runs) Len() int { return len(rs) } +func (rs Runs) Swap(i, j int) { rs[i], rs[j] = rs[j], rs[i] } +func (rs Runs) Less(i, j int) bool { + a, b := rs[i], rs[j] + if a.Backend < b.Backend { + return true + } else if a.Backend > b.Backend { + return false + } + if a.Remote < b.Remote { + return true + } else if a.Remote > b.Remote { + return false + } + if a.Path < b.Path { + return true + } else if a.Path > b.Path { + return false + } + if !a.FastList && b.FastList { + return true + } else if a.FastList && !b.FastList { + return false + } + return false +} + +// dumpOutput prints the error output +func (r *Run) dumpOutput() { + fs.Log(nil, "------------------------------------------------------------") + fs.Logf(nil, "---- %q ----", r.CmdString) + fs.Log(nil, string(r.output)) + fs.Log(nil, "------------------------------------------------------------") +} + +// trie for storing runs +type trie map[string]trie + +// turn a trie into multiple regexp matches +// +// We can't ever have a / in a regexp as it doesn't work. +func match(current trie) []string { + var names []string + var parts []string + for name, value := range current { + matchName := "^" + name + "$" + if len(value) == 0 { + names = append(names, name) + } else { + for _, part := range match(value) { + parts = append(parts, matchName+"/"+part) + } + } + } + sort.Strings(names) + if len(names) > 1 { + parts = append(parts, "^("+strings.Join(names, "|")+")$") + } else if len(names) == 1 { + parts = append(parts, "^"+names[0]+"$") + } + sort.Strings(parts) + return parts +} + +// This converts a slice of test names into a regexp which matches +// them. +func testsToRegexp(tests []string) string { + split := trie{} + // Make a trie showing which parts are used at each level + for _, test := range tests { + parent := split + for name := range strings.SplitSeq(test, "/") { + current := parent[name] + if current == nil { + current = trie{} + parent[name] = current + } + parent = current + } + } + parts := match(split) + return strings.Join(parts, "|") +} + +var failRe = regexp.MustCompile(`(?m)^\s*--- FAIL: (Test.*?) \(`) + +// findFailures looks for all the tests which failed +func (r *Run) findFailures() { + oldFailedTests := r.FailedTests + r.FailedTests = nil + excludeParents := map[string]struct{}{} + ignored := 0 + for _, matches := range failRe.FindAllSubmatch(r.output, -1) { + failedTest := string(matches[1]) + // Skip any ignored failures + if _, found := r.Ignore[failedTest]; found { + ignored++ + } else { + r.FailedTests = append(r.FailedTests, failedTest) + } + // Find all the parents of this test + parts := strings.Split(failedTest, "/") + for i := len(parts) - 1; i >= 1; i-- { + excludeParents[strings.Join(parts[:i], "/")] = struct{}{} + } + } + // Exclude the parents + newTests := r.FailedTests[:0] + for _, failedTest := range r.FailedTests { + if _, excluded := excludeParents[failedTest]; !excluded { + newTests = append(newTests, failedTest) + } + } + r.FailedTests = newTests + if len(r.FailedTests) == 0 && ignored > 0 { + fs.Logf(nil, "%q - Found %d ignored errors only - marking as good", r.CmdString, ignored) + r.err = nil + r.dumpOutput() + return + } + if len(r.FailedTests) != 0 { + r.RunFlag = testsToRegexp(r.FailedTests) + } else { + r.RunFlag = "" + } + if r.passed() && len(r.FailedTests) != 0 { + fs.Logf(nil, "%q - Expecting no errors but got: %v", r.CmdString, r.FailedTests) + r.dumpOutput() + } else if !r.passed() && len(r.FailedTests) == 0 { + fs.Logf(nil, "%q - Expecting errors but got none: %v", r.CmdString, r.FailedTests) + r.dumpOutput() + r.FailedTests = oldFailedTests + } +} + +// nextCmdLine returns the next command line +func (r *Run) nextCmdLine() []string { + CmdLine := r.CmdLine + if r.RunFlag != "" { + CmdLine = append(CmdLine, "-test.run", r.RunFlag) + } + return CmdLine +} + +// trial runs a single test +func (r *Run) trial(Opt RunOpt) { + CmdLine := r.nextCmdLine() + CmdString := toShell(CmdLine) + msg := fmt.Sprintf("%q - Starting (try %d/%d)", CmdString, r.Try, Opt.MaxTries) + fs.Log(nil, msg) + logName := path.Join(r.LogDir, r.TrialName) + out, err := os.Create(logName) + if err != nil { + fs.Fatalf(nil, "Couldn't create log file: %v", err) + } + defer func() { + err := out.Close() + if err != nil { + fs.Fatalf(nil, "Failed to close log file: %v", err) + } + }() + _, _ = fmt.Fprintln(out, msg) + + // Early exit if --try-run + if Opt.DryRun { + fs.Logf(nil, "Not executing as --dry-run: %v", CmdLine) + _, _ = fmt.Fprintln(out, "--dry-run is set - not running") + return + } + + // Start the test server if required + finish, err := testserver.Start(r.Remote) + if err != nil { + fs.Logf(nil, "%s: Failed to start test server: %v", r.Remote, err) + _, _ = fmt.Fprintf(out, "%s: Failed to start test server: %v\n", r.Remote, err) + r.err = err + return + } + defer finish() + + // Internal buffer + var b bytes.Buffer + multiOut := io.MultiWriter(out, &b) + + cmd := exec.Command(CmdLine[0], CmdLine[1:]...) + cmd.Stderr = multiOut + cmd.Stdout = multiOut + cmd.Dir = r.Path + cmd.Env = append(os.Environ(), r.Env...) + start := time.Now() + r.err = cmd.Run() + r.output = b.Bytes() + duration := time.Since(start) + r.findFailures() + if r.passed() { + msg = fmt.Sprintf("%q - Finished OK in %v (try %d/%d)", CmdString, duration, r.Try, Opt.MaxTries) + } else { + msg = fmt.Sprintf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", CmdString, duration, r.Try, Opt.MaxTries, r.err, r.FailedTests) + } + fs.Log(nil, msg) + _, _ = fmt.Fprintln(out, msg) +} + +// passed returns true if the test passed +func (r *Run) passed() bool { + return r.err == nil +} + +// GOPATH returns the current GOPATH +func GOPATH() string { + gopath := os.Getenv("GOPATH") + if gopath == "" { + gopath = build.Default.GOPATH + } + return gopath +} + +// BinaryName turns a package name into a binary name +func (r *Run) BinaryName() string { + binary := path.Base(r.Path) + ".test" + if runtime.GOOS == "windows" { + binary += ".exe" + } + return binary +} + +// BinaryPath turns a package name into a binary path +func (r *Run) BinaryPath() string { + return path.Join(r.Path, r.BinaryName()) +} + +// PackagePath returns the path to the package +func (r *Run) PackagePath() string { + return path.Join(GOPATH(), "src", r.Path) +} + +// MakeTestBinary makes the binary we will run +func (r *Run) MakeTestBinary(Opt RunOpt) { + binary := r.BinaryPath() + binaryName := r.BinaryName() + fs.Logf(nil, "%s: Making test binary %q", r.Path, binaryName) + CmdLine := []string{"go", "test", "-c"} + if Opt.Race { + CmdLine = append(CmdLine, "-race") + } + if Opt.DryRun { + fs.Logf(nil, "Not executing: %v", CmdLine) + return + } + cmd := exec.Command(CmdLine[0], CmdLine[1:]...) + cmd.Dir = r.Path + err := cmd.Run() + if err != nil { + fs.Fatalf(nil, "Failed to make test binary: %v", err) + } + if _, err := os.Stat(binary); err != nil { + fs.Fatalf(nil, "Couldn't find test binary %q", binary) + } +} + +// RemoveTestBinary removes the binary made in makeTestBinary +func (r *Run) RemoveTestBinary(Opt RunOpt) { + if Opt.DryRun { + return + } + binary := r.BinaryPath() + err := os.Remove(binary) // Delete the binary when finished + if err != nil { + fs.Logf(nil, "Error removing test binary %q: %v", binary, err) + } +} + +// Name returns the run name as a file name friendly string +func (r *Run) Name() string { + ns := []string{ + r.Backend, + strings.ReplaceAll(r.Path, "/", "."), + r.Remote, + } + if r.FastList { + ns = append(ns, "fastlist") + } + ns = append(ns, fmt.Sprintf("%d", r.Try)) + s := strings.Join(ns, "-") + s = strings.ReplaceAll(s, ":", "") + return s +} + +// Init the Run +func (r *Run) Init(Opt RunOpt) { + prefix := "-test." + if r.NoBinary { + prefix = "-" + r.CmdLine = []string{"go", "test"} + } else { + r.CmdLine = []string{"./" + r.BinaryName()} + } + testTimeout := Opt.Timeout + if r.ExtraTime > 0 { + testTimeout = time.Duration(float64(testTimeout) * r.ExtraTime) + } + r.CmdLine = append(r.CmdLine, prefix+"v", prefix+"timeout", testTimeout.String(), "-remote", r.Remote) + listRetries := Opt.ListRetries + if r.ListRetries > 0 { + listRetries = r.ListRetries + } + if listRetries > 0 { + r.CmdLine = append(r.CmdLine, "-list-retries", fmt.Sprint(listRetries)) + } + r.Try = 1 + ci := fs.GetConfig(context.Background()) + if Opt.Verbose { + r.CmdLine = append(r.CmdLine, "-verbose") + ci.LogLevel = fs.LogLevelDebug + } + if Opt.RunOnly != "" { + r.CmdLine = append(r.CmdLine, prefix+"run", Opt.RunOnly) + } + if r.FastList { + r.CmdLine = append(r.CmdLine, "-fast-list") + } + if r.Short { + r.CmdLine = append(r.CmdLine, "-short") + } + if r.SizeLimit > 0 { + r.CmdLine = append(r.CmdLine, "-size-limit", strconv.FormatInt(r.SizeLimit, 10)) + } + r.CmdString = toShell(r.CmdLine) +} + +// Logs returns all the log names +func (r *Run) Logs() []string { + return r.TrialNames +} + +// FailedTestsCSV returns the failed tests as a comma separated string, limiting the number +func (r *Run) FailedTestsCSV() string { + const maxTests = 5 + ts := r.FailedTests + if len(ts) > maxTests { + ts = ts[:maxTests:maxTests] + ts = append(ts, fmt.Sprintf("… (%d more)", len(r.FailedTests)-maxTests)) + } + return strings.Join(ts, ", ") +} + +// Run runs all the trials for this test +func (r *Run) Run(Opt RunOpt, LogDir string, result chan<- *Run) { + if r.OneOnly { + oneOnlyMu.Lock() + mu := oneOnly[r.Backend] + if mu == nil { + mu = new(sync.Mutex) + oneOnly[r.Backend] = mu + } + oneOnlyMu.Unlock() + mu.Lock() + defer mu.Unlock() + } + r.Init(Opt) + r.LogDir = LogDir + for r.Try = 1; r.Try <= Opt.MaxTries; r.Try++ { + r.TrialName = r.Name() + ".txt" + r.TrialNames = append(r.TrialNames, r.TrialName) + fs.Logf(nil, "Starting run with log %q", r.TrialName) + r.trial(Opt) + if r.passed() || r.NoRetries { + break + } + } + if !r.passed() { + r.dumpOutput() + } + result <- r +} + +// if matches then is definitely OK in the shell +var shellOK = regexp.MustCompile("^[A-Za-z0-9./_:-]+$") + +// converts an argv style input into a shell command +func toShell(args []string) (result string) { + for _, arg := range args { + if result != "" { + result += " " + } + if shellOK.MatchString(arg) { + result += arg + } else { + result += "'" + arg + "'" + } + } + return result +} diff --git a/fstest/runs/run_test.go b/fstest/runs/run_test.go new file mode 100644 index 0000000..4353b14 --- /dev/null +++ b/fstest/runs/run_test.go @@ -0,0 +1,179 @@ +package runs + +import ( + "fmt" + "os/exec" + "regexp" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTestsToRegexp(t *testing.T) { + for _, test := range []struct { + in []string + want string + }{ + { + in: []string{}, + want: "", + }, + { + in: []string{"TestOne"}, + want: "^TestOne$", + }, + { + in: []string{"TestOne", "TestTwo"}, + want: "^(TestOne|TestTwo)$", + }, + { + in: []string{"TestOne", "TestTwo", "TestThree"}, + want: "^(TestOne|TestThree|TestTwo)$", + }, + { + in: []string{"TestOne/Sub1"}, + want: "^TestOne$/^Sub1$", + }, + { + in: []string{ + "TestOne/Sub1", + "TestTwo", + }, + want: "^TestOne$/^Sub1$|^TestTwo$", + }, + { + in: []string{ + "TestOne/Sub1", + "TestOne/Sub2", + "TestTwo", + }, + want: "^TestOne$/^(Sub1|Sub2)$|^TestTwo$", + }, + { + in: []string{ + "TestOne/Sub1", + "TestOne/Sub2/SubSub1", + "TestTwo", + }, + want: "^TestOne$/^Sub1$|^TestOne$/^Sub2$/^SubSub1$|^TestTwo$", + }, + { + in: []string{ + "TestTests/A1", + "TestTests/B/B1", + "TestTests/C/C3/C31", + }, + want: "^TestTests$/^A1$|^TestTests$/^B$/^B1$|^TestTests$/^C$/^C3$/^C31$", + }, + } { + got := testsToRegexp(test.in) + assert.Equal(t, test.want, got, fmt.Sprintf("in=%v want=%q got=%q", test.in, test.want, got)) + } +} + +var runRe = regexp.MustCompile(`(?m)^\s*=== RUN\s*(Test.*?)\s*$`) + +// Test the regexp work with the -run flag in actually selecting the right tests +func TestTestsToRegexpLive(t *testing.T) { + for _, test := range []struct { + in []string + want []string + }{ + { + in: []string{ + "TestTests/A1", + "TestTests/C/C3", + }, + want: []string{ + "TestTests", + "TestTests/A1", + "TestTests/C", + "TestTests/C/C3", + "TestTests/C/C3/C31", + "TestTests/C/C3/C32", + }, + }, + { + in: []string{ + "TestTests", + "TestTests/A1", + "TestTests/B", + "TestTests/B/B1", + "TestTests/C", + }, + want: []string{ + "TestTests", + "TestTests/A1", + "TestTests/B", + "TestTests/B/B1", + "TestTests/C", + "TestTests/C/C1", + "TestTests/C/C2", + "TestTests/C/C3", + "TestTests/C/C3/C31", + "TestTests/C/C3/C32", + }, + }, + { + in: []string{ + "TestTests/A1", + "TestTests/B/B1", + "TestTests/C/C3/C31", + }, + want: []string{ + "TestTests", + "TestTests/A1", + "TestTests/B", + "TestTests/B/B1", + "TestTests/C", + "TestTests/C/C3", + "TestTests/C/C3/C31", + }, + }, + { + in: []string{ + "TestTests/B/B1", + "TestTests/C/C3/C31", + }, + want: []string{ + "TestTests", + "TestTests/B", + "TestTests/B/B1", + "TestTests/C", + "TestTests/C/C3", + "TestTests/C/C3/C31", + }, + }, + } { + runRegexp := testsToRegexp(test.in) + cmd := exec.Command("go", "test", "-v", "-run", runRegexp) + out, err := cmd.CombinedOutput() + require.NoError(t, err) + var got []string + for _, match := range runRe.FindAllSubmatch(out, -1) { + got = append(got, string(match[1])) + } + assert.Equal(t, test.want, got, fmt.Sprintf("in=%v want=%v got=%v, runRegexp=%q", test.in, test.want, got, runRegexp)) + } +} + +var nilTest = func(t *testing.T) {} + +// Nested tests for TestTestsToRegexpLive to run +func TestTests(t *testing.T) { + t.Run("A1", nilTest) + t.Run("A2", nilTest) + t.Run("B", func(t *testing.T) { + t.Run("B1", nilTest) + t.Run("B2", nilTest) + }) + t.Run("C", func(t *testing.T) { + t.Run("C1", nilTest) + t.Run("C2", nilTest) + t.Run("C3", func(t *testing.T) { + t.Run("C31", nilTest) + t.Run("C32", nilTest) + }) + }) +} diff --git a/fstest/test_all/clean.go b/fstest/test_all/clean.go new file mode 100644 index 0000000..a6f52d1 --- /dev/null +++ b/fstest/test_all/clean.go @@ -0,0 +1,86 @@ +// Clean the left over test files + +package main + +import ( + "context" + "fmt" + "regexp" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/fspath" + "github.com/rclone/rclone/fs/list" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fstest/runs" +) + +// MatchTestRemote matches the remote names used for testing (copied +// from fstest/fstest.go so we don't have to import that and get all +// its flags) +var MatchTestRemote = regexp.MustCompile(`^rclone-test-[abcdefghijklmnopqrstuvwxyz0123456789]{12,24}(_segments)?$`) + +// cleanFs runs a single clean fs for left over directories +func cleanFs(ctx context.Context, remote string, cleanup bool, Opt runs.RunOpt) error { + f, err := fs.NewFs(context.Background(), remote) + if err != nil { + return err + } + var lastErr error + if cleanup { + fs.Logf(nil, "%q - running cleanup", remote) + err = operations.CleanUp(ctx, f) + if err != nil { + lastErr = err + fs.Errorf(f, "Cleanup failed: %v", err) + } + } + entries, err := list.DirSorted(ctx, f, true, "") + if err != nil { + return err + } + err = entries.ForDirError(func(dir fs.Directory) error { + dirPath := dir.Remote() + fullPath := fspath.JoinRootPath(remote, dirPath) + if MatchTestRemote.MatchString(dirPath) { + if Opt.DryRun { + fs.Logf(nil, "Not Purging %s - -dry-run", fullPath) + return nil + } + fs.Logf(nil, "Purging %s", fullPath) + dir, err := fs.NewFs(context.Background(), fullPath) + if err != nil { + err = fmt.Errorf("NewFs failed: %w", err) + lastErr = err + fs.Errorf(fullPath, "%v", err) + return nil + } + err = operations.Purge(ctx, dir, "") + if err != nil { + err = fmt.Errorf("purge failed: %w", err) + lastErr = err + fs.Errorf(dir, "%v", err) + return nil + } + } + return nil + }) + if err != nil { + return err + } + return lastErr +} + +// cleanRemotes cleans the list of remotes passed in +func cleanRemotes(conf *runs.Config, Opt runs.RunOpt) error { + var lastError error + for _, backend := range conf.Backends { + remote := backend.Remote + fs.Logf(nil, "%q - Cleaning", remote) + err := cleanFs(context.Background(), remote, backend.CleanUp, Opt) + if err != nil { + lastError = err + fs.Logf(nil, "Failed to purge %q: %v", remote, err) + } + } + return lastError +} diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml new file mode 100644 index 0000000..24f1cfd --- /dev/null +++ b/fstest/test_all/config.yaml @@ -0,0 +1,693 @@ +tests: + - path: backend + addbackend: true + nobinary: true + short: true + - path: fs/operations + fastlist: true + - path: fs/sync + fastlist: true + - path: cmd/bisync + - path: cmd/gitannex + - path: vfs + - path: cmd/serve/restic + localonly: true + # - path: cmd/serve/docker + # localonly: true + - path: cmd/selfupdate + localonly: true +backends: + - backend: "local" + remote: "" + fastlist: false + - backend: "b2" + remote: "TestB2:" + fastlist: true + listretries: 5 + ignore: + # This test fails because B2 versions make the empty bucket undeleteable. + # It isn't possible to turn off versions, and setting hard_delete doesn't stop + # versions being made on overwrite. + - TestRmdirsNoLeaveRoot + - backend: "crypt" + remote: "TestCryptDrive:" + fastlist: true + ignore: + - TestCopyFileMaxTransfer + # - backend: "crypt" + # remote: "TestCryptSwift:" + # fastlist: false + ## chunker + - backend: "chunker" + remote: "TestChunkerLocal:" + fastlist: true + - backend: "chunker" + remote: "TestChunkerNometaLocal:" + fastlist: true + - backend: "chunker" + remote: "TestChunkerChunk3bLocal:" + fastlist: true + maxfile: 6k + - backend: "chunker" + remote: "TestChunkerChunk3bNometaLocal:" + fastlist: true + maxfile: 6k + - backend: "chunker" + remote: "TestChunkerChunk3bNoRenameLocal:" + fastlist: true + maxfile: 6k + # - backend: "chunker" + # remote: "TestChunkerMailru:" + # fastlist: true + # ignore: + # - TestApplyTransforms + # - backend: "chunker" + # remote: "TestChunkerChunk50bMailru:" + # fastlist: true + # maxfile: 10k + # ignore: + # - TestApplyTransforms + # - backend: "chunker" + # remote: "TestChunkerChunk50bYandex:" + # fastlist: true + # maxfile: 1k + # ignore: + # # Needs investigation + # - TestDeduplicateNewestByHash + # - backend: "chunker" + # remote: "TestChunkerChunk50bBox:" + # fastlist: true + # maxfile: 1k + # ignore: + # - TestIntegration/FsMkdir/FsChangeNotify + - backend: "chunker" + remote: "TestChunkerS3:" + fastlist: true + - backend: "chunker" + remote: "TestChunkerChunk50bS3:" + fastlist: true + maxfile: 1k + - backend: "chunker" + remote: "TestChunkerChunk50bMD5HashS3:" + fastlist: true + maxfile: 1k + - backend: "chunker" + remote: "TestChunkerChunk50bSHA1HashS3:" + fastlist: true + maxfile: 1k + - backend: "chunker" + remote: "TestChunkerOverCrypt:" + fastlist: true + maxfile: 6k + - backend: "chunker" + remote: "TestChunkerChunk50bMD5QuickS3:" + fastlist: true + maxfile: 1k + - backend: "chunker" + remote: "TestChunkerChunk50bSHA1QuickS3:" + fastlist: true + maxfile: 1k + ## end chunker + - backend: "cloudinary" + remote: "TestCloudinary:" + fastlist: false + ignore: + # fs/operations + - TestCheckSum + - TestCheckSumDownload + - TestHashSums/Md5 + - TestReadFile + - TestCopyURL + - TestMoveFileWithIgnoreExisting + - TestCopyFileCompareDest + #vfs + - TestFileSetModTime/cache=off,open=false,write=false + - TestFileSetModTime/cache=off,open=true,write=false + - TestRWFileHandleWriteNoWrite + ignoretests: + - cmd/gitannex + - backend: "combine" + remote: "TestCombine:dir1" + fastlist: false + ## begin compress + - backend: "compress" + remote: "TestCompress:" + fastlist: false + - backend: "compress" + remote: "TestCompressZstd:" + fastlist: false + # - backend: "compress" + # remote: "TestCompressSwift:" + # fastlist: false + - backend: "compress" + remote: "TestCompressDrive:" + fastlist: false + extratime: 2.0 + - backend: "compress" + remote: "TestCompressS3:" + fastlist: false +## end compress + - backend: "drive" + remote: "TestDrive:" + fastlist: true + ignore: + # Test with CutoffModeHard does not result in ErrorMaxTransferLimitReachedFatal + # because googleapi replaces it with a non-fatal error + - TestCopyFileMaxTransfer + - backend: "dropbox" + remote: "TestDropbox:" + fastlist: false + # - backend: "filefabric" + # remote: "TestFileFabric:" + # fastlist: false + # extratime: 2.0 + - backend: "gofile" + remote: "TestGoFile:" + fastlist: true + - backend: "filen" + remote: "TestFilen:" + fastlist: false + - backend: "filescom" + remote: "TestFilesCom:" + fastlist: false + ignoretests: + - cmd/bisync + - backend: "googlecloudstorage" + remote: "TestGoogleCloudStorage:" + fastlist: true + ignoretests: + - cmd/gitannex + - backend: "googlecloudstorage" + remote: "TestGoogleCloudStorage,directory_markers:" + fastlist: true + ignoretests: + - cmd/gitannex + - backend: "googlephotos" + remote: "TestGooglePhotos:" + tests: + - backend + - backend: "hidrive" + remote: "TestHiDrive:" + fastlist: false + - backend: "imagekit" + remote: "TestImageKit:" + fastlist: false + ignore: + - TestIntegration/FsMkdir/FsPutZeroLength + ignoretests: + - cmd/bisync + - backend: "internetarchive" + remote: "TestIA:rclone-integration-test" + fastlist: true + tests: + - backend + ignore: + - TestIntegration/FsMkdir/FsEncoding + extratime: 2.0 + - backend: "jottacloud" + remote: "TestJottacloud:" + fastlist: true + ignoretests: + - cmd/bisync + - backend: "memory" + remote: ":memory:" + fastlist: true + - backend: "netstorage" + remote: "TestnStorage:" + fastlist: true + ignoretests: + - cmd/bisync + - backend: "onedrive" + remote: "TestOneDrive:" + fastlist: false + ignore: + # This test doesn't work on a standard Onedrive account returning + # accessDenied: accountUpgradeRequired: Account Upgrade is required for this operation. + - TestIntegration/FsMkdir/FsPutFiles/PublicLink + - backend: "onedrive" + remote: "TestOneDriveBusiness:" + fastlist: false + # - backend: "onedrive" + # remote: "TestOneDriveCn:" + # fastlist: false + - backend: "s3" + remote: "TestS3:" + fastlist: true + - backend: "s3" + remote: "TestS3,directory_markers:" + fastlist: true + - backend: "s3" + remote: "TestS3Rclone:" + fastlist: true + ignoretests: + - cmd/bisync + - cmd/gitannex + - backend: "s3" + remote: "TestS3Minio:" + fastlist: true + ignore: + - TestIntegration/FsMkdir/FsPutFiles/SetTier + - TestIntegration/FsMkdir/FsEncoding/control_chars + - TestIntegration/FsMkdir/FsEncoding/leading_LF + - TestIntegration/FsMkdir/FsEncoding/leading_VT + - TestIntegration/FsMkdir/FsEncoding/punctuation + - TestIntegration/FsMkdir/FsEncoding/trailing_LF + - TestIntegration/FsMkdir/FsEncoding/trailing_VT + - backend: "s3" + remote: "TestS3MinioEdge:" + fastlist: true + ignore: + - TestIntegration/FsMkdir/FsPutFiles/SetTier + ignoretests: + - cmd/gitannex + - backend: "s3" + remote: "TestS3Wasabi:" + fastlist: true + ignore: + - TestIntegration/FsMkdir/FsEncoding/leading_CR + - TestIntegration/FsMkdir/FsEncoding/leading_LF + - TestIntegration/FsMkdir/FsEncoding/leading_HT + - TestIntegration/FsMkdir/FsEncoding/leading_VT + - TestIntegration/FsMkdir/FsPutFiles/FsPutStream/0 + - backend: "s3" + remote: "TestS3GCS:" + fastlist: true + ignore: + - TestIntegration/FsMkdir/FsEncoding/control_chars + - TestIntegration/FsMkdir/FsEncoding/leading_CR + - TestIntegration/FsMkdir/FsEncoding/leading_LF + - TestIntegration/FsMkdir/FsEncoding/trailing_CR + - TestIntegration/FsMkdir/FsEncoding/trailing_LF + - TestIntegration/FsMkdir/FsPutFiles/PublicLink + - TestIntegration/FsMkdir/FsPutFiles/SetTier + - TestIntegration/FsMkdir/FsPutFiles/Internal/Metadata/GzipEncoding + - TestIntegration/FsMkdir/FsPutFiles/Internal/Versions/VersionAt/AfterDelete/List + - TestIntegration/FsMkdir/FsPutFiles/Internal/Versions/VersionAt/AfterDelete/NewObject + - TestIntegration/FsMkdir/FsPutFiles/Internal/Versions/VersionAt/AfterTwo/List + - TestIntegration/FsMkdir/FsPutFiles/Internal/Versions/VersionAt/AfterTwo/NewObject + - TestBisyncRemoteRemote/extended_filenames + # Disabled due to excessive rate limiting at DO which cause the tests never to pass + # This hits the rate limit as documented here: https://www.digitalocean.com/docs/spaces/#limits + # 2 COPYs per 5 minutes on any individual object in a Space + # - backend: "s3" + # remote: "TestS3DigitalOcean:" + # fastlist: true + # ignore: + # - TestIntegration/FsMkdir/FsPutFiles/FsCopy + # - TestIntegration/FsMkdir/FsPutFiles/SetTier + # - backend: "s3" + # remote: "TestS3Ceph:" + # fastlist: true + # ignore: + # - TestIntegration/FsMkdir/FsPutFiles/FsCopy + # - TestIntegration/FsMkdir/FsPutFiles/SetTier + - backend: "s3" + remote: "TestS3Alibaba:" + fastlist: true + # - backend: "s3" + # remote: "TestS3Qiniu:" + # fastlist: true + # ignore: + # - TestIntegration/FsMkdir/FsEncoding/control_chars + # - TestIntegration/FsMkdir/FsEncoding/leading_VT + # - TestIntegration/FsMkdir/FsEncoding/trailing_VT + # - TestIntegration/FsMkdir/FsPutFiles/FromRoot/ListR + # - TestIntegration/FsMkdir/FsPutFiles/SetTier + # - TestIntegration/FsMkdir/FsPutFiles/FsPutStream/0 + # - TestIntegration/FsMkdir/FsPutFiles/Internal/Metadata + - backend: "s3" + remote: "TestS3R2:" + fastlist: true + ignore: + - TestIntegration/FsMkdir/FsPutFiles/SetTier + # - backend: "s3" + # remote: "TestS3FlashBlade:" + # fastlist: true + - backend: "sftp" + remote: "TestSFTPOpenssh:" + fastlist: false + ignoretests: + - cmd/gitannex + - backend: "sftp" + remote: "TestSFTPRclone:" + fastlist: false + ignoretests: + - cmd/gitannex + - backend: "sftp" + remote: "TestSFTPRcloneSSH:" + fastlist: false + ignoretests: + - cmd/gitannex + - backend: "sftp" + remote: "TestSFTPRsyncNet:" + fastlist: false + ignore: + - TestIntegration/FsMkdir/FsEncoding/trailing_space + - TestIntegration/FsMkdir/FsEncoding/trailing_CR + - TestIntegration/FsMkdir/FsEncoding/trailing_LF + - TestIntegration/FsMkdir/FsEncoding/trailing_HT + - TestIntegration/FsMkdir/FsEncoding/trailing_VT + - TestIntegration/FsMkdir/FsEncoding/trailing_dot + - TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8 + - TestIntegration/FsMkdir/FsEncoding/URL_encoding + ignoretests: + - cmd/bisync + - backend: "sugarsync" + remote: "TestSugarSync:Test" + fastlist: false + ignore: + - TestIntegration/FsMkdir/FsPutFiles/PublicLink + - backend: "swift" + remote: "TestSwiftAIO:" + fastlist: true + ignoretests: + - cmd/gitannex + - backend: "swift" + remote: "TestSwiftAIOsegments:" + fastlist: true + ignoretests: + - cmd/gitannex + # - backend: "swift" + # remote: "TestSwift:" + # fastlist: true + # ignoretests: + # - cmd/bisync + # - backend: "swift" + # remote: "TestSwiftCeph:" + # fastlist: true + # ignore: + # - TestIntegration/FsMkdir/FsPutFiles/FsCopy + - backend: "yandex" + remote: "TestYandex:" + fastlist: false + - backend: "ftp" + remote: "TestFTPProftpd:" + fastlist: false + ignoretests: + - cmd/gitannex + - backend: "ftp" + remote: "TestFTPPureftpd:" + fastlist: false + ignoretests: + - cmd/gitannex + - backend: "ftp" + remote: "TestFTPVsftpd:" + fastlist: false + ignoretests: + - cmd/gitannex + - backend: "ftp" + remote: "TestFTPRclone:" + fastlist: false + ignoretests: + - cmd/gitannex + - backend: "box" + remote: "TestBox:" + fastlist: false + - backend: "fichier" + remote: "TestFichier:" + fastlist: false + listretries: 5 + tests: + - backend + # Disabled due to account cancellation - no longer available to me + # - backend: "qingstor" + # remote: "TestQingStor:" + # fastlist: false + # oneonly: true + # cleanup: true + # ignore: + # # This test fails because of a broken bucket in the account! + # - TestIntegration/FsMkdir/FsPutFiles/FromRoot/ListR + - backend: "azureblob" + remote: "TestAzureBlob:" + fastlist: true + ignore: + # It just isn't possible to preserve the existing file with azure blob + # and make sure we don't leak uncommitted blocks. + - TestMultithreadCopyAbort + - backend: "azureblob" + remote: "TestAzureBlob,directory_markers:" + fastlist: true + ignore: + # It just isn't possible to preserve the existing file with azure blob + # and make sure we don't leak uncommitted blocks. + - TestMultithreadCopyAbort + - backend: "azurefiles" + remote: "TestAzureFiles:" + - backend: "pcloud" + remote: "TestPcloud:" + fastlist: true + - backend: "pikpak" + remote: "TestPikPak:" + fastlist: false + ignore: + # fs/operations + - TestCheckSum + - TestHashSums/Md5 + # fs/sync + - TestSyncWithTrackRenames + # integration + - TestIntegration/FsMkdir/FsPutFiles/ObjectMimeType + # This test fails with message + # "share_status_prohibited" (9): Sorry, the sharing service is under maintenance in the current region. + - TestIntegration/FsMkdir/FsPutFiles/PublicLink + ignoretests: + - cmd/bisync + - cmd/gitannex + - backend: "pixeldrain" + remote: "TestPixeldrain:" + ignore: + - TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8 + fastlist: false + ignoretests: + - cmd/bisync + - backend: "webdav" + remote: "TestWebdavNextcloud:" + ignore: + - TestIntegration/FsMkdir/FsEncoding/punctuation + - TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8 + fastlist: false + ignoretests: + - cmd/bisync + - cmd/gitannex + - backend: "webdav" + remote: "TestWebdavOwncloud:" + ignore: + - TestIntegration/FsMkdir/FsEncoding/punctuation + - TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8 + fastlist: false + ignoretests: + - cmd/bisync + - cmd/gitannex + - backend: "webdav" + remote: "TestWebdavInfiniteScale:" + ignore: + - TestIntegration/FsMkdir/FsEncoding/punctuation + - TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8 + env: + - RCLONE_NO_CHECK_CERTIFICATE=true + fastlist: false + ignoretests: + - cmd/bisync + - cmd/gitannex + - backend: "webdav" + remote: "TestWebdavRclone:" + ignore: + - TestFileReadAtZeroLength + fastlist: false + ignoretests: + - cmd/gitannex + - backend: "cache" + remote: "TestCache:" + fastlist: false + ignoretests: + - cmd/bisync + - backend: "mega" + remote: "TestMega:" + fastlist: false + ignore: + - TestIntegration/FsMkdir/FsPutFiles/PublicLink + - TestDirRename + - TestFileRename + ignoretests: + - cmd/bisync + - backend: "opendrive" + remote: "TestOpenDrive:" + fastlist: false + ignoretests: + - cmd/bisync + - backend: "union" + remote: "TestUnion:" + fastlist: false + - backend: "koofr" + remote: "TestKoofr:" + fastlist: false + # - backend: "koofr" + # remote: "TestDigiStorage:" + # fastlist: false + - backend: "linkbox" + remote: "TestLinkbox:" + fastlist: false + ignore: + - TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8 + - TestRWFileHandleWriteNoWrite + - TestCaseInsensitiveMoveFile + - TestFixCase + - TestListDirSorted # Can't upload files starting with . - FIXME fix with encoding + - TestSyncOverlapWithFilter # Can't upload files starting with . - FIXME fix with encoding + ignoretests: + - cmd/bisync + - cmd/gitannex + - backend: "premiumizeme" + remote: "TestPremiumizeMe:" + fastlist: false + - backend: "protondrive" + remote: "TestProtonDrive:" + fastlist: false + ignoretests: + - cmd/bisync + - cmd/gitannex + - backend: "putio" + remote: "TestPutio:" + fastlist: false + extratime: 2.0 + ignore: + - TestIntegration/FsMkdir/FsEncoding/URL_encoding + ignoretests: + - cmd/bisync + # - backend: "sharefile" + # remote: "TestSharefile:" + # fastlist: false + - backend: "sia" + remote: "TestSia:" + fastlist: false + ignoretests: + - cmd/gitannex + - backend: "mailru" + remote: "TestMailru:" + subdir: false + fastlist: false + oneonly: true + ignore: + - TestApplyTransforms + ignoretests: + - cmd/bisync + - backend: "seafile" + remote: "TestSeafileV6:" + fastlist: false + ignore: + - TestIntegration/FsMkdir/FsPutFiles/FsDirMove + ignoretests: + - cmd/gitannex + - backend: "seafile" + remote: "TestSeafile:" + fastlist: true + ignoretests: + - cmd/gitannex + - backend: "seafile" + remote: "TestSeafileEncrypted:" + fastlist: true + ignoretests: + - cmd/gitannex + - backend: "smb" + remote: "TestSMB:rclone" + fastlist: false + ignoretests: + - cmd/gitannex + - backend: "smb" + remote: "TestSMBKerberos:rclone" + fastlist: false + env: + - KRB5_CONFIG=/tmp/rclone_krb5/krb5.conf + - KRB5CCNAME=/tmp/rclone_krb5/ccache + ignoretests: + - cmd/gitannex + - backend: "smb" + remote: "TestSMBKerberosCcache:rclone" + fastlist: false + env: + - KRB5_CONFIG=/tmp/rclone_krb5_ccache/krb5.conf + ignoretests: + - cmd/gitannex + - backend: "storj" + remote: "TestStorj:" + fastlist: true + ignoretests: + - cmd/bisync + - backend: "zoho" + remote: "TestZoho:" + fastlist: false + extratime: 2.0 + tests: + - backend + - backend: "hdfs" + remote: "TestHdfs:" + fastlist: false + ignore: + - TestSyncUTFNorm + ignoretests: + - cmd/gitannex + - backend: "oracleobjectstorage" + remote: "TestOracleObjectStorage:" + fastlist: true + ignore: + - TestIntegration/FsMkdir/FsEncoding/control_chars + - TestIntegration/FsMkdir/FsEncoding/leading_CR + - TestIntegration/FsMkdir/FsEncoding/leading_LF + - TestIntegration/FsMkdir/FsEncoding/trailing_CR + - TestIntegration/FsMkdir/FsEncoding/trailing_LF + - TestIntegration/FsMkdir/FsEncoding/leading_HT + - TestIntegration/FsMkdir/FsPutFiles/FsPutStream/0 + ignoretests: + - cmd/bisync + - cmd/gitannex + - backend: "quatrix" + remote: "TestQuatrix:" + fastlist: false + ignoretests: + - cmd/bisync + - backend: "ulozto" + remote: "TestUlozto:" + fastlist: false + # - backend: "iclouddrive" + # remote: "TestICloudDrive:" + # fastlist: false + - backend: "filelu" + remote: "TestFileLu:" + fastlist: false + ignore: + - TestRWFileHandleWriteNoWrite + ignoretests: + - cmd/bisync + - cmd/gitannex + - backend: "shade" + remote: "TestShade:" + fastlist: false + + - backend: "archive" + remote: "TestArchive:" + fastlist: false + ignoretests: + - cmd/bisync + - cmd/gitannex + ignore: + # These are caused by the archive backend returning the underlying objects + # with the parent backend having a different precision. + - TestServerSideCopyOverSelf + - TestServerSideMoveOverSelf + - backend: "internxt" + remote: "TestInternxt:" + fastlist: false + listretries: 5 + ignore: + - TestRWFileHandleWriteNoWrite + - backend: "drime" + remote: "TestDrime:" + ignoretests: + # The TestBisyncRemoteLocal/check_access_filters tests fail due to duplicated objects + - cmd/bisync + fastlist: false + extratime: 2.0 + - backend: "studip" + remote: "TestStudIP:" + fastlist: false diff --git a/fstest/test_all/test_all.go b/fstest/test_all/test_all.go new file mode 100644 index 0000000..ae1c73a --- /dev/null +++ b/fstest/test_all/test_all.go @@ -0,0 +1,149 @@ +// Run tests for all the remotes. Run this with package names which +// need integration testing. +// +// See the `test` target in the Makefile. +package main + +/* FIXME + +Make TesTrun have a []string of flags to try - that then makes it generic + +*/ + +import ( + "encoding/csv" + "flag" + "fmt" + "math/rand" + "os" + "path" + "strings" + "time" + + _ "github.com/rclone/rclone/backend/all" // import all fs + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configfile" + "github.com/rclone/rclone/fstest/runs" + "github.com/rclone/rclone/lib/pacer" +) + +func init() { + // Flags + flag.IntVar(&Opt.MaxTries, "maxtries", 5, "Number of times to try each test") + flag.IntVar(&Opt.MaxN, "n", 20, "Maximum number of tests to run at once") + flag.StringVar(&Opt.TestRemotes, "remotes", "", "Comma separated list of remotes to test, e.g. 'TestSwift:,TestS3'") + flag.StringVar(&Opt.TestBackends, "backends", "", "Comma separated list of backends to test, e.g. 's3,googlecloudstorage") + flag.StringVar(&Opt.TestTests, "tests", "", "Comma separated list of tests to test, e.g. 'fs/sync,fs/operations'") + flag.BoolVar(&Opt.Clean, "clean", false, "Instead of testing, clean all left over test directories") + flag.StringVar(&Opt.RunOnly, "run", "", "Run only those tests matching the regexp supplied") + flag.DurationVar(&Opt.Timeout, "timeout", 60*time.Minute, "Maximum time to run each test for before giving up") + flag.BoolVar(&Opt.Race, "race", false, "If set run the tests under the race detector") + flag.StringVar(&Opt.ConfigFile, "config", "fstest/test_all/config.yaml", "Path to config file") + flag.StringVar(&Opt.OutputDir, "output", path.Join(os.TempDir(), "rclone-integration-tests"), "Place to store results") + flag.StringVar(&Opt.EmailReport, "email", "", "Set to email the report to the address supplied") + flag.BoolVar(&Opt.DryRun, "dry-run", false, "Print commands which would be executed only") + flag.StringVar(&Opt.URLBase, "url-base", "https://pub.rclone.org/integration-tests/", "Base for the online version") + flag.StringVar(&Opt.UploadPath, "upload", "", "Set this to an rclone path to upload the results here") + flag.BoolVar(&Opt.Verbose, "verbose", false, "Set to enable verbose logging in the tests") + flag.IntVar(&Opt.ListRetries, "list-retries", -1, "Number or times to retry listing - set to override the default") +} + +var Opt = &runs.RunOpt{} + +func main() { + flag.Parse() + conf, err := runs.NewConfig(Opt.ConfigFile) + if err != nil { + fs.Log(nil, "test_all should be run from the root of the rclone source code") + fs.Fatal(nil, fmt.Sprint(err)) + } + configfile.Install() + + // Seed the random number generator + randInstance := rand.New(rand.NewSource(time.Now().UTC().UnixNano())) + + // Filter selection + if Opt.TestRemotes != "" { + // CSV parse to support connection string remotes with commas like -remotes local,\"TestGoogleCloudStorage,directory_markers:\" + r := csv.NewReader(strings.NewReader(Opt.TestRemotes)) + remotes, err := r.Read() + if err != nil { + fs.Fatalf(Opt.TestRemotes, "error CSV-parsing -remotes string: %v", err) + } + fs.Debugf(Opt.TestRemotes, "using remotes: %v", remotes) + conf.FilterBackendsByRemotes(remotes) + } + if Opt.TestBackends != "" { + conf.FilterBackendsByBackends(strings.Split(Opt.TestBackends, ",")) + } + if Opt.TestTests != "" { + conf.FilterTests(strings.Split(Opt.TestTests, ",")) + } + + // Just clean the directories if required + if Opt.Clean { + err := cleanRemotes(conf, *Opt) + if err != nil { + fs.Fatalf(nil, "Failed to clean: %v", err) + } + return + } + + var names []string + for _, remote := range conf.Backends { + names = append(names, remote.Remote) + } + fs.Logf(nil, "Testing remotes: %s", strings.Join(names, ", ")) + + // Runs we will do for this test in random order + testRuns := conf.MakeRuns() + randInstance.Shuffle(len(testRuns), testRuns.Swap) + + // Create Report + report := runs.NewReport(*Opt) + + // Make the test binaries, one per Path found in the tests + done := map[string]struct{}{} + for _, run := range testRuns { + if _, found := done[run.Path]; !found { + done[run.Path] = struct{}{} + if !run.NoBinary { + run.MakeTestBinary(*Opt) + defer run.RemoveTestBinary(*Opt) + } + } + } + + // workaround for cache backend as we run simultaneous tests + _ = os.Setenv("RCLONE_CACHE_DB_WAIT_TIME", "30m") + + // start the tests + results := make(chan *runs.Run, len(testRuns)) + awaiting := 0 + tokens := pacer.NewTokenDispenser(Opt.MaxN) + for _, run := range testRuns { + tokens.Get() + go func(run *runs.Run) { + defer tokens.Put() + run.Run(*Opt, report.LogDir, results) + }(run) + awaiting++ + } + + // Wait for the tests to finish + for ; awaiting > 0; awaiting-- { + t := <-results + report.RecordResult(t) + } + + // Log and exit + report.End() + report.LogSummary() + report.LogJSON() + report.LogHTML() + report.EmailHTML(*Opt) + report.Upload(*Opt) + if !report.AllPassed() { + os.Exit(1) + } +} diff --git a/fstest/testserver/images/test-hdfs/Dockerfile b/fstest/testserver/images/test-hdfs/Dockerfile new file mode 100644 index 0000000..07c621d --- /dev/null +++ b/fstest/testserver/images/test-hdfs/Dockerfile @@ -0,0 +1,45 @@ +# A very minimal hdfs server for integration testing rclone +FROM debian:stretch + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openjdk-8-jdk \ + net-tools curl python krb5-user krb5-kdc krb5-admin-server \ + && rm -rf /var/lib/apt/lists/* + +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ + +ENV HADOOP_VERSION 3.2.1 +ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz +RUN set -x \ + && curl -fSL "$HADOOP_URL" -o /tmp/hadoop.tar.gz \ + && tar -xvf /tmp/hadoop.tar.gz -C /opt/ \ + && rm /tmp/hadoop.tar.gz* + +RUN ln -s /opt/hadoop-$HADOOP_VERSION/etc/hadoop /etc/hadoop +RUN mkdir /opt/hadoop-$HADOOP_VERSION/logs + +RUN mkdir /hadoop-data +RUN mkdir -p /hadoop/dfs/name +RUN mkdir -p /hadoop/dfs/data + +ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION +ENV HADOOP_CONF_DIR=/etc/hadoop +ENV MULTIHOMED_NETWORK=1 + +ENV USER=root +ENV PATH $HADOOP_HOME/bin/:$PATH + +ADD core-site.xml /etc/hadoop/core-site.xml +ADD hdfs-site.xml /etc/hadoop/hdfs-site.xml +ADD httpfs-site.xml /etc/hadoop/httpfs-site.xml +ADD kms-site.xml /etc/hadoop/kms-site.xml +ADD mapred-site.xml /etc/hadoop/mapred-site.xml +ADD yarn-site.xml /etc/hadoop/yarn-site.xml + +ADD krb5.conf /etc/ +ADD kdc.conf /etc/krb5kdc/ +RUN echo '*/admin@KERBEROS.RCLONE *' > /etc/krb5kdc/kadm5.acl + +ADD run.sh /run.sh +RUN chmod a+x /run.sh +CMD ["/run.sh"] diff --git a/fstest/testserver/images/test-hdfs/README.md b/fstest/testserver/images/test-hdfs/README.md new file mode 100644 index 0000000..1f760dd --- /dev/null +++ b/fstest/testserver/images/test-hdfs/README.md @@ -0,0 +1,57 @@ +# Test HDFS + +This is a docker image for rclone's integration tests which runs an +hdfs filesystem in a docker image. + +## Build + +``` +docker build --rm -t rclone/test-hdfs . +docker push rclone/test-hdfs +``` + +# Test + +configure remote: +``` +[TestHdfs] +type = hdfs +namenode = 127.0.0.1:8020 +username = root +``` + +run tests +``` +cd backend/hdfs +GO111MODULE=on go test -v +``` + +hdfs logs will be available in `.stdout.log` and `.stderr.log` + +# Kerberos + +test can be run against kerberos-enabled hdfs + +1. configure local krb5.conf + ``` + [libdefaults] + default_realm = KERBEROS.RCLONE + [realms] + KERBEROS.RCLONE = { + kdc = localhost + } + ``` + +2. enable kerberos in remote configuration + ``` + [TestHdfs] + ... + service_principal_name = hdfs/localhost + data_transfer_protection = privacy + ``` + +3. run test + ``` + cd backend/hdfs + KERBEROS=true GO111MODULE=on go test -v + ```
\ No newline at end of file diff --git a/fstest/testserver/images/test-hdfs/core-site.xml b/fstest/testserver/images/test-hdfs/core-site.xml new file mode 100644 index 0000000..061d48d --- /dev/null +++ b/fstest/testserver/images/test-hdfs/core-site.xml @@ -0,0 +1,12 @@ +<configuration> + <property><name>fs.defaultFS</name><value>hdfs://localhost:8020</value></property> + <property><name>hadoop.http.staticuser.user</name><value>root</value></property> + <property><name>hadoop.proxyuser.root.groups</name><value>root,nogroup</value></property> + <property><name>hadoop.proxyuser.root.hosts</name><value>*</value></property> + <!-- KERBEROS BEGIN --> + <property><name>hadoop.security.authentication</name><value>kerberos</value></property> + <property><name>hadoop.security.authorization</name><value>true</value></property> + <property><name>hadoop.rpc.protection</name><value>integrity</value></property> + <property><name>hadoop.user.group.static.mapping.overrides</name><value>user=supergroup</value></property> + <!-- KERBEROS END --> +</configuration> diff --git a/fstest/testserver/images/test-hdfs/hdfs-site.xml b/fstest/testserver/images/test-hdfs/hdfs-site.xml new file mode 100644 index 0000000..3f3f3a6 --- /dev/null +++ b/fstest/testserver/images/test-hdfs/hdfs-site.xml @@ -0,0 +1,31 @@ +<configuration> + <property><name>dfs.client.use.datanode.hostname</name><value>true</value></property> + <property><name>dfs.datanode.data.dir</name><value>file:///hadoop/dfs/data</value></property> + <property><name>dfs.datanode.use.datanode.hostname</name><value>true</value></property> + <property><name>dfs.namenode.accesstime.precision</name><value>3600000</value></property> + <property><name>dfs.namenode.http-bind-host</name><value>0.0.0.0</value></property> + <property><name>dfs.namenode.https-bind-host</name><value>0.0.0.0</value></property> + <property><name>dfs.namenode.name.dir</name><value>file:///hadoop/dfs/name</value></property> + <property><name>dfs.namenode.rpc-bind-host</name><value>0.0.0.0</value></property> + <property><name>dfs.namenode.safemode.extension</name><value>5000</value></property> + <property><name>dfs.namenode.servicerpc-bind-host</name><value>0.0.0.0</value></property> + <property><name>dfs.replication</name><value>2</value></property> + <property><name>nfs.dump.dir</name><value>/tmp</value></property> + <!-- KERBEROS BEGIN --> + <property><name>ignore.secure.ports.for.testing</name><value>true</value></property> + <property><name>dfs.safemode.extension</name><value>0</value></property> + <property><name>dfs.block.access.token.enable</name><value>true</value></property> + + <property><name>dfs.encrypt.data.transfer</name><value>true</value></property> + <property><name>dfs.encrypt.data.transfer.algorithm</name><value>rc4</value></property> + <property><name>dfs.encrypt.data.transfer.cipher.suites</name><value>AES/CTR/NoPadding</value></property> + + <property><name>dfs.namenode.kerberos.principal</name> <value>hdfs/_HOST@KERBEROS.RCLONE</value></property> + <property><name>dfs.web.authentication.kerberos.principal</name><value>HTTP/_HOST@KERBEROS.RCLONE</value></property> + <property><name>dfs.datanode.kerberos.principal</name> <value>hdfs/_HOST@KERBEROS.RCLONE</value></property> + + <property><name>dfs.namenode.keytab.file</name> <value>/etc/hadoop/kerberos.key</value></property> + <property><name>dfs.web.authentication.kerberos.keytab</name><value>/etc/hadoop/kerberos.key</value></property> + <property><name>dfs.datanode.keytab.file</name> <value>/etc/hadoop/kerberos.key</value></property> + <!-- KERBEROS END --> +</configuration> diff --git a/fstest/testserver/images/test-hdfs/httpfs-site.xml b/fstest/testserver/images/test-hdfs/httpfs-site.xml new file mode 100644 index 0000000..8313843 --- /dev/null +++ b/fstest/testserver/images/test-hdfs/httpfs-site.xml @@ -0,0 +1,2 @@ +<configuration> +</configuration> diff --git a/fstest/testserver/images/test-hdfs/kdc.conf b/fstest/testserver/images/test-hdfs/kdc.conf new file mode 100644 index 0000000..9eeb0bb --- /dev/null +++ b/fstest/testserver/images/test-hdfs/kdc.conf @@ -0,0 +1,4 @@ +[realms] + KERBEROS.RCLONE = { + acl_file = /etc/krb5kdc/kadm5.acl + } diff --git a/fstest/testserver/images/test-hdfs/kms-site.xml b/fstest/testserver/images/test-hdfs/kms-site.xml new file mode 100644 index 0000000..8313843 --- /dev/null +++ b/fstest/testserver/images/test-hdfs/kms-site.xml @@ -0,0 +1,2 @@ +<configuration> +</configuration> diff --git a/fstest/testserver/images/test-hdfs/krb5.conf b/fstest/testserver/images/test-hdfs/krb5.conf new file mode 100644 index 0000000..012950b --- /dev/null +++ b/fstest/testserver/images/test-hdfs/krb5.conf @@ -0,0 +1,10 @@ +[libdefaults] + default_realm = KERBEROS.RCLONE + dns_lookup_realm = false + dns_lookup_kdc = false + forwardable = true + proxiable = true +[realms] + KERBEROS.RCLONE = { + kdc = localhost + } diff --git a/fstest/testserver/images/test-hdfs/mapred-site.xml b/fstest/testserver/images/test-hdfs/mapred-site.xml new file mode 100644 index 0000000..9f70286 --- /dev/null +++ b/fstest/testserver/images/test-hdfs/mapred-site.xml @@ -0,0 +1,5 @@ +<configuration> + <property><name>mapreduce.framework.name</name><value>yarn</value></property> + <property><name>yarn.nodemanager.bind-host</name><value>0.0.0.0</value></property> +</configuration> + diff --git a/fstest/testserver/images/test-hdfs/run.sh b/fstest/testserver/images/test-hdfs/run.sh new file mode 100755 index 0000000..207e11a --- /dev/null +++ b/fstest/testserver/images/test-hdfs/run.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +KERBEROS=${KERBEROS-"false"} + +if [ $KERBEROS = "true" ]; then + echo prepare kerberos + ADMIN_PASSWORD="kerberos" + USER_PASSWORD="user" + + echo -e "$ADMIN_PASSWORD\n$ADMIN_PASSWORD" | kdb5_util -r "KERBEROS.RCLONE" create -s + echo -e "$ADMIN_PASSWORD\n$ADMIN_PASSWORD" | kadmin.local -q "addprinc hadoop/admin" + echo -e "$USER_PASSWORD\n$USER_PASSWORD" | kadmin.local -q "addprinc user" + kadmin.local -q 'addprinc -randkey hdfs/localhost' + kadmin.local -q 'addprinc -randkey hdfs/rclone-hdfs' + kadmin.local -q 'addprinc -randkey HTTP/localhost' + kadmin.local -p hadoop/admin -q "ktadd -k /etc/hadoop/kerberos.key hdfs/localhost hdfs/rclone-hdfs HTTP/localhost" + service krb5-kdc restart + echo -e "$USER_PASSWORD\n" | kinit user + klist + echo kerberos ready +else + echo drop kerberos from configuration files + sed -i '/KERBEROS BEGIN/,/KERBEROS END/d' /etc/hadoop/core-site.xml + sed -i '/KERBEROS BEGIN/,/KERBEROS END/d' /etc/hadoop/hdfs-site.xml +fi + + +echo format namenode +hdfs namenode -format test + +hdfs namenode & +hdfs datanode & +exec sleep infinity diff --git a/fstest/testserver/images/test-hdfs/yarn-site.xml b/fstest/testserver/images/test-hdfs/yarn-site.xml new file mode 100644 index 0000000..ade8c7f --- /dev/null +++ b/fstest/testserver/images/test-hdfs/yarn-site.xml @@ -0,0 +1,14 @@ +<configuration> + <property><name>yarn.log-aggregation-enable</name><value>true</value></property> + <property><name>yarn.log.server.url</name><value>http://localhost:8188/applicationhistory/logs/</value></property> + <property><name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name><value>org.apache.hadoop.mapred.ShuffleHandler</value></property> + <property><name>yarn.nodemanager.aux-services</name><value>mapreduce_shuffle</value></property> + <property><name>yarn.nodemanager.bind-host</name><value>0.0.0.0</value></property> + <property><name>yarn.nodemanager.bind-host</name><value>0.0.0.0</value></property> + <property><name>yarn.nodemanager.remote-app-log-dir</name><value>/app-logs</value></property> + <property><name>yarn.timeline-service.bind-host</name><value>0.0.0.0</value></property> + <property><name>yarn.timeline-service.enabled</name><value>true</value></property> + <property><name>yarn.timeline-service.generic-application-history.enabled</name><value>true</value></property> + <property><name>yarn.timeline-service.hostname</name><value>historyserver.hadoop</value></property> + <property><name>yarn.timeline-service.leveldb-timeline-store.path</name><value>/hadoop/yarn/timeline</value></property> +</configuration> diff --git a/fstest/testserver/images/test-sftp-openssh/Dockerfile b/fstest/testserver/images/test-sftp-openssh/Dockerfile new file mode 100644 index 0000000..f7a8455 --- /dev/null +++ b/fstest/testserver/images/test-sftp-openssh/Dockerfile @@ -0,0 +1,11 @@ +# A very minimal sftp server for integration testing rclone +FROM alpine:latest + +# User rclone, password password +RUN \ + apk add openssh && \ + ssh-keygen -A && \ + adduser -D rclone && \ + echo "rclone:password" | chpasswd + +ENTRYPOINT [ "/usr/sbin/sshd", "-D" ] diff --git a/fstest/testserver/images/test-sftp-openssh/README.md b/fstest/testserver/images/test-sftp-openssh/README.md new file mode 100644 index 0000000..2e98b44 --- /dev/null +++ b/fstest/testserver/images/test-sftp-openssh/README.md @@ -0,0 +1,17 @@ +# Test SFTP Openssh + +This is a docker image for rclone's integration tests which runs an +openssh server in a docker image. + +## Build + +``` +docker build --rm -t rclone/test-sftp-openssh . +docker push rclone/test-sftp-openssh +``` + +# Test + +``` +rclone lsf -R --sftp-host 172.17.0.2 --sftp-user rclone --sftp-pass $(rclone obscure password) :sftp: +``` diff --git a/fstest/testserver/init.d/PORTS.md b/fstest/testserver/init.d/PORTS.md new file mode 100644 index 0000000..fc955a2 --- /dev/null +++ b/fstest/testserver/init.d/PORTS.md @@ -0,0 +1,49 @@ +# Ports for tests + +All these tests need to run on a different port. + +They should be bound to localhost so they are not accessible externally. + +| Port | Test | +|:-----:|:----:| +| 88 | TestHdfs | +| 750 | TestHdfs | +| 8020 | TestHdfs | +| 8086 | TestSeafileV6 | +| 8087 | TestSeafile | +| 8088 | TestSeafileEncrypted | +| 9866 | TestHdfs | +| 28620 | TestWebdavRclone | +| 28621 | TestSFTPRclone | +| 28622 | TestFTPRclone | +| 28623 | TestSFTPRcloneSSH | +| 28624 | TestS3Rclone | +| 28625 | TestS3Minio | +| 28626 | TestS3MinioEdge | +| 28627 | TestSFTPOpenssh | +| 28628 | TestSwiftAIO | +| 28629 | TestWebdavNextcloud | +| 28630 | TestSMB | +| 28631 | TestFTPProftpd | +| 28632 | TestSwiftAIOsegments | +| 28633 | TestSMBKerberos | +| 28634 | TestSMBKerberos | +| 28635 | TestS3Exaba | +| 28636 | TestS3Exaba | +| 28637 | TestSMBKerberosCcache | +| 28638 | TestSMBKerberosCcache | +| 28639 | TestWebdavInfiniteScale | +| 38081 | TestWebdavOwncloud | + +## Non localhost tests + +All these use `$(docker_ip)` which means they don't work on macOS or +Windows. It is proabably possible to make them work with some effort +but will require port forwarding a range of ports and configuring the +FTP server to only use that range of ports. The FTP server will likely +need know it is behind a NAT so it advertises the correct external IP. + +- TestFTPProftpd +- TestFTPPureftpd +- TestFTPVsftpd +- TestFTPVsftpdTLS diff --git a/fstest/testserver/init.d/README.md b/fstest/testserver/init.d/README.md new file mode 100644 index 0000000..c9acd92 --- /dev/null +++ b/fstest/testserver/init.d/README.md @@ -0,0 +1,48 @@ +This directory contains scripts to start and stop servers for testing. + +The commands are named after the remotes in use. They are executable +files with the following parameters: + + start - starts the server if not running + stop - stops the server if nothing is using it + status - returns non-zero exit code if the server is not running + reset - stops the server and resets any reference counts + +These will be called automatically by test_all if that remote is +required. + +When start is run it should output config parameters for that remote. +If a `_connect` parameter is output then that will be used for a +connection test. For example if `_connect=127.0.0.1:80` then a TCP +connection will be made to `127.0.0.1:80` and only when that succeeds +will the test continue. + +If in addition to `_connect`, `_connect_delay=5s` is also present then +after the connection succeeds rclone will wait `5s` before continuing. +This is for servers that aren't quite ready even though they have +opened their TCP ports. + +## Writing new scripts + +A docker based server or an `rclone serve` based server should be easy +to write. Look at one of the examples. + +`run.bash` contains boilerplate to be included in a bash script for +interpreting the command line parameters. This does reference counting +to ensure multiple copies of the server aren't running at once. +Including this is mandatory. It will call your `start()`, `stop()` and +`status()` functions. + +`docker.bash` contains library functions to help with docker +implementations. It contains implementations of `stop()` and +`status()` so all you have to do is write a `start()` function. + +`rclone-serve.bash` contains functions to help with `rclone serve` +based implementations. It contains implementations of `stop()` and +`status()` so all you have to do is write a `start()` function which +should call the `run()` function provided. + +Any external TCP or UDP ports used should be unique as any of the +servers might be running together. So please create a new line in the +[PORTS](PORTS.md) file to allocate your server a port. Bind any ports +to localhost so they aren't accessible externally. diff --git a/fstest/testserver/init.d/TestFTPProftpd b/fstest/testserver/init.d/TestFTPProftpd new file mode 100755 index 0000000..029909a --- /dev/null +++ b/fstest/testserver/init.d/TestFTPProftpd @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -e + +NAME=proftpd +USER=rclone +PASS=RaidedBannedPokes5 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "FTP_USERNAME=rclone" \ + -e "FTP_PASSWORD=$PASS" \ + hauptmedia/proftpd + + echo type=ftp + echo host=$(docker_ip) + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo encoding=Asterisk,Ctl,Dot,Slash + echo _connect=$(docker_ip):21 +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestFTPPureftpd b/fstest/testserver/init.d/TestFTPPureftpd new file mode 100755 index 0000000..69c9285 --- /dev/null +++ b/fstest/testserver/init.d/TestFTPPureftpd @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -e + +NAME=pureftpd +USER=rclone +PASS=AcridSpiesBooks2 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "FTP_USER_NAME=rclone" \ + -e "FTP_USER_PASS=$PASS" \ + -e "FTP_USER_HOME=/data" \ + -e "FTP_MAX_CLIENTS=50" \ + -e "FTP_MAX_CONNECTIONS=50" \ + -e "FTP_PASSIVE_PORTS=30000:40000" \ + stilliard/pure-ftpd + + echo type=ftp + echo host=$(docker_ip) + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo encoding=BackSlash,Ctl,Del,Dot,RightSpace,Slash,SquareBracket + echo _connect=$(docker_ip):21 +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestFTPRclone b/fstest/testserver/init.d/TestFTPRclone new file mode 100755 index 0000000..85ad26a --- /dev/null +++ b/fstest/testserver/init.d/TestFTPRclone @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +NAME=rclone-serve-ftp +USER=rclone +PASS=FuddleIdlingJell5 +IP=127.0.0.1 +PORT=28622 + +start() { + run rclone serve ftp --user $USER --pass $PASS --addr ${IP}:${PORT} ${DATADIR} + + echo type=ftp + echo host=${IP} + echo port=$PORT + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo _connect=${IP}:${PORT} +} + +. $(dirname "$0")/rclone-serve.bash diff --git a/fstest/testserver/init.d/TestFTPVsftpd b/fstest/testserver/init.d/TestFTPVsftpd new file mode 100755 index 0000000..d33dcf0 --- /dev/null +++ b/fstest/testserver/init.d/TestFTPVsftpd @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -e + +NAME=vsftpd +USER=rclone +PASS=TiffedRestedSian4 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "FTP_USER=rclone" \ + -e "FTP_PASS=$PASS" \ + fauria/vsftpd + + echo type=ftp + echo host=$(docker_ip) + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo writing_mdtm=true + echo encoding=Ctl,LeftPeriod,Slash + echo _connect=$(docker_ip):21 +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestFTPVsftpdTLS b/fstest/testserver/init.d/TestFTPVsftpdTLS new file mode 100755 index 0000000..ebcd3b0 --- /dev/null +++ b/fstest/testserver/init.d/TestFTPVsftpdTLS @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -e + +NAME=vsftpdtls +USER=rclone +PASS=TiffedRestedSian4 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "FTP_USER=rclone" \ + -e "FTP_PASS=$PASS" \ + rclone/vsftpd + + echo type=ftp + echo host=$(docker_ip) + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo writing_mdtm=true + echo encoding=Ctl,LeftPeriod,Slash + echo _connect=$(docker_ip):21 +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestHdfs b/fstest/testserver/init.d/TestHdfs new file mode 100755 index 0000000..26aea1a --- /dev/null +++ b/fstest/testserver/init.d/TestHdfs @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -e + +NAME=rclone-hdfs +KERBEROS=${KERBEROS-"false"} + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name "rclone-hdfs" \ + --hostname "rclone-hdfs" \ + -e "KERBEROS=$KERBEROS" \ + -p 127.0.0.1:9866:9866 \ + -p 127.0.0.1:8020:8020 \ + -p 127.0.0.1:750:750 \ + -p 127.0.0.1:88:88 \ + rclone/test-hdfs + sleep 30 + + if [ $KERBEROS = "true" ]; then + docker cp rclone-hdfs:/tmp/krb5cc_0 /tmp/krb5cc_`id -u` + fi + + echo type=hdfs + echo namenode=127.0.0.1:8020 + echo username=root + echo _connect=127.0.0.1:8020 +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestS3Exaba b/fstest/testserver/init.d/TestS3Exaba new file mode 100755 index 0000000..c9a7f92 --- /dev/null +++ b/fstest/testserver/init.d/TestS3Exaba @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -e + +NAME=exaba +USER="Use the webui to find the access_key_id" +PASS="Use the webui to find the secret_access_key" +PORT=28635 +WEBUIPORT=28636 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "CLUSTER_NAME=rclone" \ + -e "CLUSTER_SIZE_GB=20" \ + -p 127.0.0.1:${PORT}:9000 \ + -p 127.0.0.1:${WEBUIPORT}:9006 \ + exaba/exaba + + echo type=s3 + echo provider=Exaba + echo access_key_id=$USER + echo secret_access_key=$PASS + echo endpoint=http://127.0.0.1:${PORT}/ + echo webui=http://127.0.0.1:${WEBUIPORT}/ + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestS3Minio b/fstest/testserver/init.d/TestS3Minio new file mode 100755 index 0000000..b4d3dde --- /dev/null +++ b/fstest/testserver/init.d/TestS3Minio @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -e + +NAME=minio +USER=rclone +PASS=AxedBodedGinger7 +PORT=28625 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "MINIO_ACCESS_KEY=$USER" \ + -e "MINIO_SECRET_KEY=$PASS" \ + -p 127.0.0.1:${PORT}:9000 \ + minio/minio server /data + + echo type=s3 + echo provider=Minio + echo access_key_id=$USER + echo secret_access_key=$PASS + echo endpoint=http://127.0.0.1:${PORT}/ + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestS3MinioEdge b/fstest/testserver/init.d/TestS3MinioEdge new file mode 100755 index 0000000..399ec7f --- /dev/null +++ b/fstest/testserver/init.d/TestS3MinioEdge @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -e + +NAME=minio-edge +USER=rclone +PASS=DeniseOxygenEiffel4 +PORT=28626 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "MINIO_ACCESS_KEY=$USER" \ + -e "MINIO_SECRET_KEY=$PASS" \ + -p 127.0.0.1:${PORT}:9000 \ + minio/minio:edge server /data + + echo type=s3 + echo provider=Minio + echo access_key_id=$USER + echo secret_access_key=$PASS + echo endpoint=http://127.0.0.1:${PORT}/ + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestS3Rclone b/fstest/testserver/init.d/TestS3Rclone new file mode 100755 index 0000000..c336322 --- /dev/null +++ b/fstest/testserver/init.d/TestS3Rclone @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +NAME=rclone-serve-s3 +ACCESS_KEY_ID=rclone +SECRET_ACCESS_KEY=JoltRogueVerde5 +IP=127.0.0.1 +PORT=28624 + +start() { + run rclone serve s3 --auth-key ${ACCESS_KEY_ID},${SECRET_ACCESS_KEY} --addr ${IP}:${PORT} ${DATADIR} + + echo type=s3 + echo provider=Rclone + echo endpoint=http://${IP}:${PORT}/ + echo access_key_id=${ACCESS_KEY_ID} + echo secret_access_key=${SECRET_ACCESS_KEY} + echo _connect=${IP}:${PORT} +} + +. $(dirname "$0")/rclone-serve.bash diff --git a/fstest/testserver/init.d/TestSFTPOpenssh b/fstest/testserver/init.d/TestSFTPOpenssh new file mode 100755 index 0000000..91a9c9a --- /dev/null +++ b/fstest/testserver/init.d/TestSFTPOpenssh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -e + +NAME=rclone-sftp-openssh +USER=rclone +PASS=password +PORT=28627 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name ${NAME} \ + -p 127.0.0.1:${PORT}:22 \ + rclone/test-sftp-openssh + + echo type=sftp + echo host=127.0.0.1 + echo port=$PORT + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo copy_is_hardlink=true + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSFTPRclone b/fstest/testserver/init.d/TestSFTPRclone new file mode 100755 index 0000000..f553112 --- /dev/null +++ b/fstest/testserver/init.d/TestSFTPRclone @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +NAME=rclone-serve-sftp +USER=rclone +PASS=CranesBallotDorsey5 +IP=127.0.0.1 +PORT=28621 + +start() { + run rclone serve sftp --user $USER --pass $PASS --addr ${IP}:${PORT} ${DATADIR} + + echo type=sftp + echo host=${IP} + echo port=$PORT + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo _connect=${IP}:${PORT} +} + +. $(dirname "$0")/rclone-serve.bash diff --git a/fstest/testserver/init.d/TestSFTPRcloneSSH b/fstest/testserver/init.d/TestSFTPRcloneSSH new file mode 100755 index 0000000..989a5e6 --- /dev/null +++ b/fstest/testserver/init.d/TestSFTPRcloneSSH @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -e + +NAME=rclone-serve-sftp-ssh +IP=127.0.0.1 +PORT=28623 +PRIVATE_KEY=/tmp/${NAME}.key +PUBLIC_KEY=/tmp/${NAME}.pub + +start() { + +cat >${PRIVATE_KEY} <<'#EOF' +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEAv7e6d+AbxELvRk7sZipketuqgE4/vVbf/6PMuOd1OSPyOOsAOs41 +tvc4Sk4S+/6ReHW4l1DKy5IH0smOsA1k58kKFkN1NChHU5z0CitAiZwdl7zqvxNJqlMmYi +GTubhQdMnDrq0AAnhyr9TFrcZmYPcp9tHcpt9VQWeLYR/16tT53WnpgTuMWlgyM58bpCh/ +cDO7gOjSyXHhPPxrU1qdr5g/5T9HFgQfi2CX8vk4pDY+Qw1Lnp1MMpKT4i9xWGMU8oDJG3 +08RrzUi9tz1RTePtbs4xXOy8cXOZaAODDQok4iWvJEpGgJYLjhNuHzZiUDcfc1SkXvONui +7j5RC/rsQOYB5Sd7ATlF4HymAxZJ3iPu+eYBZi7lwIPeug+42WlVon+D5dOYmrgcPpAZv7 +67Lthv62FMmvc1SHHGPZLS3dWfbZeXayve9+wIkKFEuDN76zYAavjSRm9fBKny6J+noJgp +bDMVNnTfNA28fsNbsCS6OsBjLbiFjMHxhuYACMaVAAAFgBfF8CkXxfApAAAAB3NzaC1yc2 +EAAAGBAL+3unfgG8RC70ZO7GYqZHrbqoBOP71W3/+jzLjndTkj8jjrADrONbb3OEpOEvv+ +kXh1uJdQysuSB9LJjrANZOfJChZDdTQoR1Oc9AorQImcHZe86r8TSapTJmIhk7m4UHTJw6 +6tAAJ4cq/Uxa3GZmD3KfbR3KbfVUFni2Ef9erU+d1p6YE7jFpYMjOfG6Qof3Azu4Do0slx +4Tz8a1Nana+YP+U/RxYEH4tgl/L5OKQ2PkMNS56dTDKSk+IvcVhjFPKAyRt9PEa81Ivbc9 +UU3j7W7OMVzsvHFzmWgDgw0KJOIlryRKRoCWC44Tbh82YlA3H3NUpF7zjbou4+UQv67EDm +AeUnewE5ReB8pgMWSd4j7vnmAWYu5cCD3roPuNlpVaJ/g+XTmJq4HD6QGb++uy7Yb+thTJ +r3NUhxxj2S0t3Vn22Xl2sr3vfsCJChRLgze+s2AGr40kZvXwSp8uifp6CYKWwzFTZ03zQN +vH7DW7AkujrAYy24hYzB8YbmAAjGlQAAAAMBAAEAAAGABOxf8oIj1Gdvo5uVQI5oJCuN9l +uMEX2wpOz87earwPrmVoXabKgtAvTYUjgtDqGb9L75LZGak529a7FXY7gEVlt4UdgLo3pB +UqleLwCrWJ1UuTfVw3BoXOJjwvNfys4r6sPfrZWtwWJ8d318UhkdOfI+9qKvCu4DT3msP6 +NFenFbtU7p+zKfSRaou2CjohSUKTp63zWbbCbrhNhqnSpfkEnVojp8xdj3QmoJnOi/hqAJ ++0jVH06kzUusVounWoC41pTr1Dlnvy+gWhJcZtHNBXixL6JCM9XGh+z0XFgO8YTiHMdTfz +Q2wf06TdXzOcM6XwPn3azyKmk7sn2v0s1pxGw8eu8tbmdU46xaLijwipn/B7NMsjk2gnqN +eptwb/SQmIZZcloQZYLx9PejarAHe2NJ5BSJqOrSHZHYXjiSKj4X55lGdDOVCUCf4lmStQ +qCS2LiM8Uhfga6f3X5EIBY75kqzmovDnPrqjufnCfYjBzQZ/m/txCbnZ9sTdQfXoVFAAAA +wQC/5nbU5HzZtg7bA3kfBRUNGUSl8nM2zENY9Rxc8sZiL7iH3s1HAVyz8Frvmc1Wgt6EF+ +WhtmNFklOmdYwq0W5+2qRdUN2P9QL+GKbuyp4AvwRmCFNhgm2GrCWQj/rkZ61vYS3bM8J8 +MNJglvU2FktXvwFODhf6Kv/7fZQnJCf2LTMG6hIKF4LdBOSS/0V5MH2v4xu2U64wqQAQnu +KzG4sRedsSHBGSknROJ7eGvGPZLh56PRb2gYPItoHcTMHqB6UAAADBAM4Xv6tHQFZtL+ul +FwVVKhr3EKGY3+RV9IBXvXDkhee4i594Yl67BFUSU4eDb4xuek24znwKn+UFERzp+1X02h +I1dZRdKtzJWOUQIF0FMPHbaPTuS7viT1OrL/PnG1yXUa+ii0qLExYI9qe463e8w6fNwhaT +Em2wiDZcxr8SjQfqimyfmDizVlLE/xdgJ56eJ+OyXLjpezKKQif9YcFUW/eHej3YEcok6o +WDwYpXG8z+VwOnOV4UN/sS8pLkJUdpqwAAAMEA7iTUVH3IvXMno0xYrVdhgNtZZGfQnryt +pRfG/f5eQ1tjEQwE31mrbBcR278YWlQZwrMWZ0hDaJ3Q/Cp0+JySlm17jsA33lnTRmCHF4 +WolX/LlFtH6jLr9SB8GOsn/8lC6IcvkED0UYYBjXipl6Unh9ZPnpbmJK9SgZWKNTGS1NBw +xcVWIZTKOrpCD+zWH6KCviuhd3J5vBgkSVxTUzDqA7TmnMnUUxDQAAaU7Eqtt3CJuxsJIs +vmZ2QrVK8TstC/AAAACm5jd0Bkb2dnZXI= +-----END OPENSSH PRIVATE KEY----- +#EOF +chmod 600 ${PRIVATE_KEY} + +cat >${PUBLIC_KEY} <<'#EOF' +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/t7p34BvEQu9GTuxmKmR626qATj+9Vt//o8y453U5I/I46wA6zjW29zhKThL7/pF4dbiXUMrLkgfSyY6wDWTnyQoWQ3U0KEdTnPQKK0CJnB2XvOq/E0mqUyZiIZO5uFB0ycOurQACeHKv1MWtxmZg9yn20dym31VBZ4thH/Xq1PndaemBO4xaWDIznxukKH9wM7uA6NLJceE8/GtTWp2vmD/lP0cWBB+LYJfy+TikNj5DDUuenUwykpPiL3FYYxTygMkbfTxGvNSL23PVFN4+1uzjFc7Lxxc5loA4MNCiTiJa8kSkaAlguOE24fNmJQNx9zVKRe8426LuPlEL+uxA5gHlJ3sBOUXgfKYDFkneI+755gFmLuXAg966D7jZaVWif4Pl05iauBw+kBm/vrsu2G/rYUya9zVIccY9ktLd1Z9tl5drK9737AiQoUS4M3vrNgBq+NJGb18EqfLon6egmClsMxU2dN80Dbx+w1uwJLo6wGMtuIWMwfGG5gAIxpU= user@rclone-serve-test +#EOF +chmod 600 ${PUBLIC_KEY} + + run rclone serve sftp --authorized-keys "${PUBLIC_KEY}" --addr ${IP}:${PORT} ${DATADIR} + + echo type=sftp + echo ssh=ssh -i ${PRIVATE_KEY} -o StrictHostKeyChecking=no -p ${PORT} user@${IP} + echo _connect=${IP}:${PORT} +} + +. $(dirname "$0")/rclone-serve.bash diff --git a/fstest/testserver/init.d/TestSMB b/fstest/testserver/init.d/TestSMB new file mode 100755 index 0000000..4e10d59 --- /dev/null +++ b/fstest/testserver/init.d/TestSMB @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -e + +NAME=smb +USER=rclone +PASS=GNF3Cqeu +WORKGROUP=thepub +PORT=28630 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -p 127.0.0.1:${PORT}:445 \ + -p 127.0.0.1:${PORT}:445/udp \ + dperson/samba \ + -p \ + -u "rclone;${PASS}" \ + -w "${WORKGROUP}" \ + -s "public;/share" \ + -s "rclone;/rclone;yes;no;no;rclone" + + echo type=smb + echo host=127.0.0.1 + echo user=$USER + echo port=$PORT + echo pass=$(rclone obscure $PASS) + echo domain=$WORKGROUP + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSMBKerberos b/fstest/testserver/init.d/TestSMBKerberos new file mode 100755 index 0000000..f54eeb6 --- /dev/null +++ b/fstest/testserver/init.d/TestSMBKerberos @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -e + +IMAGE=rclone/test-smb-kerberos +NAME=smb-kerberos +USER=rclone +DOMAIN=RCLONE +REALM=RCLONE.LOCAL +SMB_PORT=28633 +KRB5_PORT=28634 + +# KRB5_CONFIG and KRB5CCNAME should be set by the caller but default +# them here for the integration tests +export TEMP_DIR=/tmp/rclone_krb5 +mkdir -p "${TEMP_DIR}" +export KRB5_CONFIG=${KRB5_CONFIG:-${TEMP_DIR}/krb5.conf} +export KRB5CCNAME=${KRB5CCNAME:-${TEMP_DIR}/ccache} + +. $(dirname "$0")/docker.bash + +start() { + docker build -t ${IMAGE} --load - <<EOF +FROM alpine:3.21 +RUN apk add --no-cache samba-dc +RUN rm -rf /etc/samba/smb.conf /var/lib/samba \ + && mkdir -p /var/lib/samba/private \ + && samba-tool domain provision \ + --use-rfc2307 \ + --option acl_xattr:security_acl_name=user.NTACL \ + --realm=$REALM \ + --domain=$DOMAIN \ + --server-role=dc \ + --dns-backend=SAMBA_INTERNAL \ + --host-name=localhost \ + && samba-tool user add --random-password $USER \ + && samba-tool user setexpiry $USER --noexpiry \ + && mkdir -m 777 /share /rclone \ + && cat <<EOS >> /etc/samba/smb.conf +[global] +server signing = auto +[public] +path = /share +browseable = yes +read only = yes +guest ok = yes +[rclone] +path = /rclone +browseable = yes +read only = no +guest ok = no +valid users = rclone +EOS +CMD ["samba", "-i"] +EOF + + docker run --rm -d --name ${NAME} \ + -p 127.0.0.1:${SMB_PORT}:445 \ + -p 127.0.0.1:${SMB_PORT}:445/udp \ + -p 127.0.0.1:${KRB5_PORT}:88 \ + ${IMAGE} + + # KRB5_CONFIG and KRB5CCNAME are set by the caller + cat > ${KRB5_CONFIG} <<EOF +[libdefaults] + default_realm = ${REALM} +[realms] +${REALM} = { + kdc = localhost +} +EOF + docker cp ${KRB5_CONFIG} ${NAME}:/etc/krb5.conf + docker exec ${NAME} samba-tool user get-kerberos-ticket rclone --output-krb5-ccache=/tmp/ccache + docker cp ${NAME}:/tmp/ccache ${KRB5CCNAME} + sed -i -e "s/localhost/localhost:${KRB5_PORT}/" ${KRB5_CONFIG} + + echo type=smb + echo host=localhost + echo port=$SMB_PORT + echo use_kerberos=true + echo _connect=127.0.0.1:${SMB_PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSMBKerberosCcache b/fstest/testserver/init.d/TestSMBKerberosCcache new file mode 100755 index 0000000..1238299 --- /dev/null +++ b/fstest/testserver/init.d/TestSMBKerberosCcache @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +set -e + +# Set default location for Kerberos config and ccache. Can be overridden by the caller +# using environment variables RCLONE_TEST_CUSTOM_CCACHE_LOCATION and KRB5_CONFIG. +export TEMP_DIR=/tmp/rclone_krb5_ccache +mkdir -p "${TEMP_DIR}" +export KRB5_CONFIG=${KRB5_CONFIG:-${TEMP_DIR}/krb5.conf} +export RCLONE_TEST_CUSTOM_CCACHE_LOCATION=${RCLONE_TEST_CUSTOM_CCACHE_LOCATION:-${TEMP_DIR}/ccache} + +IMAGE=rclone/test-smb-kerberos-ccache +NAME=smb-kerberos-ccache +USER=rclone +DOMAIN=RCLONE +REALM=RCLONE.LOCAL +SMB_PORT=28637 +KRB5_PORT=28638 + +. $(dirname "$0")/docker.bash + +start() { + docker build -t ${IMAGE} --load - <<EOF +FROM alpine:3.21 +RUN apk add --no-cache samba-dc +RUN rm -rf /etc/samba/smb.conf /var/lib/samba \ + && mkdir -p /var/lib/samba/private \ + && samba-tool domain provision \ + --use-rfc2307 \ + --option acl_xattr:security_acl_name=user.NTACL \ + --realm=$REALM \ + --domain=$DOMAIN \ + --server-role=dc \ + --dns-backend=SAMBA_INTERNAL \ + --host-name=localhost \ + && samba-tool user add --random-password $USER \ + && samba-tool user setexpiry $USER --noexpiry \ + && mkdir -m 777 /share /rclone \ + && cat <<EOS >> /etc/samba/smb.conf +[global] +server signing = auto +[public] +path = /share +browseable = yes +read only = yes +guest ok = yes +[rclone] +path = /rclone +browseable = yes +read only = no +guest ok = no +valid users = rclone +EOS +CMD ["samba", "-i"] +EOF + + docker run --rm -d --name ${NAME} \ + -p 127.0.0.1:${SMB_PORT}:445 \ + -p 127.0.0.1:${SMB_PORT}:445/udp \ + -p 127.0.0.1:${KRB5_PORT}:88 \ + ${IMAGE} + + cat > "${KRB5_CONFIG}" <<EOF +[libdefaults] + default_realm = ${REALM} +[realms] +${REALM} = { + kdc = localhost +} +EOF + + docker cp "${KRB5_CONFIG}" ${NAME}:/etc/krb5.conf + docker exec ${NAME} samba-tool user get-kerberos-ticket rclone --output-krb5-ccache=/tmp/ccache + docker cp ${NAME}:/tmp/ccache "${RCLONE_TEST_CUSTOM_CCACHE_LOCATION}" + sed -i -e "s/localhost/localhost:${KRB5_PORT}/" "${KRB5_CONFIG}" + + echo type=smb + echo host=localhost + echo port=$SMB_PORT + echo use_kerberos=true + echo kerberos_ccache=${RCLONE_TEST_CUSTOM_CCACHE_LOCATION} + echo _connect=127.0.0.1:${SMB_PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSeafile b/fstest/testserver/init.d/TestSeafile new file mode 100755 index 0000000..553e9d0 --- /dev/null +++ b/fstest/testserver/init.d/TestSeafile @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -e + +# environment variables passed on docker-compose +export NAME=seafile7 +export MYSQL_ROOT_PASSWORD=pixenij4zacoguq0kopamid6 +export SEAFILE_ADMIN_EMAIL=seafile@rclone.org +export SEAFILE_ADMIN_PASSWORD=pixenij4zacoguq0kopamid6 +export SEAFILE_IP=127.0.0.1 +export SEAFILE_PORT=8087 +export SEAFILE_TEST_DATA=${SEAFILE_TEST_DATA:-/tmp/seafile-test-data} +export SEAFILE_VERSION=latest + +# make sure the data directory exists +mkdir -p ${SEAFILE_TEST_DATA}/${NAME} + +# docker-compose project directory +COMPOSE_DIR=$(dirname "$0")/seafile + +start() { + docker-compose --project-directory ${COMPOSE_DIR} --project-name ${NAME} --file ${COMPOSE_DIR}/docker-compose.yml up -d + + # wait for Seafile server to start + seafile_endpoint="http://${SEAFILE_IP}:${SEAFILE_PORT}/" + wait_seconds=1 + echo -n "Waiting for Seafile server to start" + for iterations in `seq 1 60`; + do + http_code=$(curl -s -o /dev/null -L -w '%{http_code}' "$seafile_endpoint" || true;) + if [ "$http_code" -eq 200 ]; then + echo + break + fi + echo -n "." + sleep $wait_seconds + done + + # authentication token answer should be like: {"token":"dbf58423f1632b5b679a13b0929f1d0751d9250c"} + TOKEN=`curl --silent \ + --data-urlencode username=${SEAFILE_ADMIN_EMAIL} -d password=${SEAFILE_ADMIN_PASSWORD} \ + http://${SEAFILE_IP}:${SEAFILE_PORT}/api2/auth-token/ \ + | sed 's/^{"token":"\(.*\)"}$/\1/'` + + # create default library + curl --silent -o /dev/null -X POST -H "Authorization: Token ${TOKEN}" "http://${SEAFILE_IP}:${SEAFILE_PORT}/api2/default-repo/" + + echo _connect=${SEAFILE_IP}:${SEAFILE_PORT} + echo type=seafile + echo url=http://${SEAFILE_IP}:${SEAFILE_PORT}/ + echo user=${SEAFILE_ADMIN_EMAIL} + echo pass=$(rclone obscure ${SEAFILE_ADMIN_PASSWORD}) + echo library=My Library +} + +stop() { + if status ; then + docker-compose --project-directory ${COMPOSE_DIR} --project-name ${NAME} --file ${COMPOSE_DIR}/docker-compose.yml down + fi +} + +status() { + if docker ps --format "{{.Names}}" | grep ^${NAME}_seafile_1$ >/dev/null ; then + echo "$NAME running" + else + echo "$NAME not running" + return 1 + fi + return 0 +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSeafileEncrypted b/fstest/testserver/init.d/TestSeafileEncrypted new file mode 100755 index 0000000..dd4b6bc --- /dev/null +++ b/fstest/testserver/init.d/TestSeafileEncrypted @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -e + +# local variables +TEST_LIBRARY=Encrypted +TEST_LIBRARY_PASSWORD=SecretKey + +# environment variables passed on docker-compose +export NAME=seafile7encrypted +export MYSQL_ROOT_PASSWORD=pixenij4zacoguq0kopamid6 +export SEAFILE_ADMIN_EMAIL=seafile@rclone.org +export SEAFILE_ADMIN_PASSWORD=pixenij4zacoguq0kopamid6 +export SEAFILE_IP=127.0.0.1 +export SEAFILE_PORT=8088 +export SEAFILE_TEST_DATA=${SEAFILE_TEST_DATA:-/tmp/seafile-test-data} +export SEAFILE_VERSION=latest + +# make sure the data directory exists +mkdir -p ${SEAFILE_TEST_DATA}/${NAME} + +# docker-compose project directory +COMPOSE_DIR=$(dirname "$0")/seafile + +start() { + docker-compose --project-directory ${COMPOSE_DIR} --project-name ${NAME} --file ${COMPOSE_DIR}/docker-compose.yml up -d + + # it takes some time for the database to be created + sleep 60 + + # authentication token answer should be like: {"token":"dbf58423f1632b5b679a13b0929f1d0751d9250c"} + TOKEN=`curl --silent \ + --data-urlencode username=${SEAFILE_ADMIN_EMAIL} -d password=${SEAFILE_ADMIN_PASSWORD} \ + http://${SEAFILE_IP}:${SEAFILE_PORT}/api2/auth-token/ \ + | sed 's/^{"token":"\(.*\)"}$/\1/'` + + # create encrypted library + curl --silent -o /dev/null -X POST -d "name=${TEST_LIBRARY}&passwd=${TEST_LIBRARY_PASSWORD}" -H "Authorization: Token ${TOKEN}" "http://${SEAFILE_IP}:${SEAFILE_PORT}/api2/repos/" + + echo _connect=${SEAFILE_IP}:${SEAFILE_PORT} + echo type=seafile + echo url=http://${SEAFILE_IP}:${SEAFILE_PORT}/ + echo user=${SEAFILE_ADMIN_EMAIL} + echo pass=$(rclone obscure ${SEAFILE_ADMIN_PASSWORD}) + echo library=${TEST_LIBRARY} + echo library_key=$(rclone obscure ${TEST_LIBRARY_PASSWORD}) +} + +stop() { + if status ; then + docker-compose --project-directory ${COMPOSE_DIR} --project-name ${NAME} --file ${COMPOSE_DIR}/docker-compose.yml down + fi +} + +status() { + if docker ps --format "{{.Names}}" | grep ^${NAME}_seafile_1$ >/dev/null ; then + echo "$NAME running" + else + echo "$NAME not running" + return 1 + fi + return 0 +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSeafileV6 b/fstest/testserver/init.d/TestSeafileV6 new file mode 100755 index 0000000..e2365aa --- /dev/null +++ b/fstest/testserver/init.d/TestSeafileV6 @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +set -e + +# local variables +NAME=seafile6 +SEAFILE_IP=127.0.0.1 +SEAFILE_PORT=8086 +SEAFILE_ADMIN_EMAIL=seafile@rclone.org +SEAFILE_ADMIN_PASSWORD=qebiwob7wafixif8sojiboj4 +SEAFILE_TEST_DATA=${SEAFILE_TEST_DATA:-/tmp/seafile-test-data} +SEAFILE_VERSION=latest + +. $(dirname "$0")/docker.bash + +start() { + # make sure the data directory exists + mkdir -p ${SEAFILE_TEST_DATA}/${NAME} + + docker run --rm -d --name $NAME \ + -e SEAFILE_SERVER_HOSTNAME=${SEAFILE_IP}:${SEAFILE_PORT} \ + -e SEAFILE_ADMIN_EMAIL=${SEAFILE_ADMIN_EMAIL} \ + -e SEAFILE_ADMIN_PASSWORD=${SEAFILE_ADMIN_PASSWORD} \ + -v ${SEAFILE_TEST_DATA}/${NAME}:/shared \ + -p ${SEAFILE_IP}:${SEAFILE_PORT}:80 \ + seafileltd/seafile:${SEAFILE_VERSION} + + # it takes some time for the database to be created + sleep 60 + + # authentication token answer should be like: {"token":"dbf58423f1632b5b679a13b0929f1d0751d9250c"} + TOKEN=`curl --silent \ + --data-urlencode username=${SEAFILE_ADMIN_EMAIL} -d password=${SEAFILE_ADMIN_PASSWORD} \ + http://${SEAFILE_IP}:${SEAFILE_PORT}/api2/auth-token/ \ + | sed 's/^{"token":"\(.*\)"}$/\1/'` + + # create default library + curl --silent -o /dev/null -X POST -H "Authorization: Token ${TOKEN}" "http://${SEAFILE_IP}:${SEAFILE_PORT}/api2/default-repo/" + + echo _connect=${SEAFILE_IP}:${SEAFILE_PORT} + echo type=seafile + echo url=http://${SEAFILE_IP}:${SEAFILE_PORT}/ + echo user=${SEAFILE_ADMIN_EMAIL} + echo pass=$(rclone obscure ${SEAFILE_ADMIN_PASSWORD}) + echo library=My Library +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSia b/fstest/testserver/init.d/TestSia new file mode 100755 index 0000000..9b11caf --- /dev/null +++ b/fstest/testserver/init.d/TestSia @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -e + +NAME=Sia + +# shellcheck disable=SC1090 +. "$(dirname "$0")"/docker.bash + +# wait until Sia test network is up, +# the Sia renter forms contracts on the blockchain +# and the renter is upload ready +wait_for_sia() { + until curl -A Sia-Agent -s "$1" | grep -q '"ready":true' + do + sleep 5 + done +} +export -f wait_for_sia + +start() { + # use non-production sia port in test + SIA_CONN="127.0.0.1:39980" + # nebulouslabs/siaantfarm is stale, use up-to-date image + ANTFARM_IMAGE=ivandeex/sia-antfarm:latest + + # pull latest antfarm image (dont use local image) + docker pull --quiet $ANTFARM_IMAGE + + # start latest antfarm with default config + docker run --rm --detach --name "$NAME" \ + --publish "${SIA_CONN}:9980" \ + $ANTFARM_IMAGE + + # wait until the test network is upload ready + timeout 300 bash -c "wait_for_sia ${SIA_CONN}/renter/uploadready" + + # confirm backend type in the generated rclone.conf + echo "type=sia" + # override keys in the Sia section of generated rclone.conf + echo "api_url=http://${SIA_CONN}/" + # hint test harness where to probe for connection + echo "_connect=${SIA_CONN}" +} + +stop() { + if status ; then + docker logs "$NAME" >> sia-test.log 2>&1 + docker kill "$NAME" + echo "${NAME} stopped" + fi +} + +# shellcheck disable=SC1090 +. "$(dirname "$0")"/run.bash diff --git a/fstest/testserver/init.d/TestSwiftAIO b/fstest/testserver/init.d/TestSwiftAIO new file mode 100755 index 0000000..7e20bd6 --- /dev/null +++ b/fstest/testserver/init.d/TestSwiftAIO @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -e + +NAME=swift-aio +PORT=28628 + +. $(dirname "$0")/docker.bash + +start() { + # We need to replace the remakerings in the container to create Policy-1. + docker run --rm -d --name ${NAME} \ + -p 127.0.0.1:${PORT}:8080 \ + -v $(dirname "$0")/TestSwiftAIO.d/remakerings:/etc/swift/remakerings:ro \ + openstackswift/saio + + echo type=swift + echo env_auth=false + echo user=test:tester + echo key=testing + echo auth=http://127.0.0.1:${PORT}/auth/v1.0 + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSwiftAIO.d/remakerings b/fstest/testserver/init.d/TestSwiftAIO.d/remakerings new file mode 100755 index 0000000..27c49b1 --- /dev/null +++ b/fstest/testserver/init.d/TestSwiftAIO.d/remakerings @@ -0,0 +1,46 @@ +#!/bin/sh + +if ! grep -q "^\[storage-policy:1\]" swift.conf; then + cat <<EOF >> swift.conf + +[storage-policy:1] +name = Policy-1 +EOF +fi + +rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz + +swift-ring-builder object.builder create 10 1 1 +swift-ring-builder object.builder add r1z1-127.0.0.1:6200/swift-d0 1 +swift-ring-builder object.builder add r1z1-127.0.0.1:6200/swift-d1 1 +swift-ring-builder object.builder add r1z1-127.0.0.1:6200/swift-d2 1 +swift-ring-builder object.builder add r1z1-127.0.0.1:6200/swift-d3 1 +swift-ring-builder object.builder add r1z1-127.0.0.1:6200/swift-d4 1 +swift-ring-builder object.builder add r1z1-127.0.0.1:6200/swift-d5 1 +swift-ring-builder object.builder rebalance +swift-ring-builder container.builder create 10 1 1 +swift-ring-builder container.builder add r1z1-127.0.0.1:6201/swift-d0 1 +swift-ring-builder container.builder add r1z1-127.0.0.1:6201/swift-d1 1 +swift-ring-builder container.builder add r1z1-127.0.0.1:6201/swift-d2 1 +swift-ring-builder container.builder add r1z1-127.0.0.1:6201/swift-d3 1 +swift-ring-builder container.builder add r1z1-127.0.0.1:6201/swift-d4 1 +swift-ring-builder container.builder add r1z1-127.0.0.1:6201/swift-d5 1 +swift-ring-builder container.builder rebalance +swift-ring-builder account.builder create 10 1 1 +swift-ring-builder account.builder add r1z1-127.0.0.1:6202/swift-d0 1 +swift-ring-builder account.builder add r1z1-127.0.0.1:6202/swift-d1 1 +swift-ring-builder account.builder add r1z1-127.0.0.1:6202/swift-d2 1 +swift-ring-builder account.builder add r1z1-127.0.0.1:6202/swift-d3 1 +swift-ring-builder account.builder add r1z1-127.0.0.1:6202/swift-d4 1 +swift-ring-builder account.builder add r1z1-127.0.0.1:6202/swift-d5 1 +swift-ring-builder account.builder rebalance + +# For Policy-1: +swift-ring-builder object-1.builder create 10 1 1 +swift-ring-builder object-1.builder add r1z1-127.0.0.1:6200/swift-d0 1 +swift-ring-builder object-1.builder add r1z1-127.0.0.1:6200/swift-d1 1 +swift-ring-builder object-1.builder add r1z1-127.0.0.1:6200/swift-d2 1 +swift-ring-builder object-1.builder add r1z1-127.0.0.1:6200/swift-d3 1 +swift-ring-builder object-1.builder add r1z1-127.0.0.1:6200/swift-d4 1 +swift-ring-builder object-1.builder add r1z1-127.0.0.1:6200/swift-d5 1 +swift-ring-builder object-1.builder rebalance diff --git a/fstest/testserver/init.d/TestSwiftAIOsegments b/fstest/testserver/init.d/TestSwiftAIOsegments new file mode 100755 index 0000000..db02630 --- /dev/null +++ b/fstest/testserver/init.d/TestSwiftAIOsegments @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +NAME=swift-aio-segments +PORT=28632 + +. $(dirname "$0")/docker.bash + +start() { + # We need to replace the remakerings in the container to create Policy-1. + docker run --rm -d --name ${NAME} \ + -p 127.0.0.1:${PORT}:8080 \ + -v $(dirname "$0")/TestSwiftAIO.d/remakerings:/etc/swift/remakerings:ro \ + openstackswift/saio + + echo type=swift + echo env_auth=false + echo user=test:tester + echo key=testing + echo auth=http://127.0.0.1:${PORT}/auth/v1.0 + echo use_segments_container=false + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestWebdavInfiniteScale b/fstest/testserver/init.d/TestWebdavInfiniteScale new file mode 100755 index 0000000..f13e22b --- /dev/null +++ b/fstest/testserver/init.d/TestWebdavInfiniteScale @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -e + +NAME=infinitescale +USER=admin +PASS=admin +PORT=28639 +CONF_DIR=/tmp/ocis-config +mkdir -p ${CONF_DIR} +chmod 777 ${CONF_DIR} || true + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm --name $NAME \ + -v ${CONF_DIR}:/etc/ocis \ + -e "OCIS_INSECURE=true" \ + -e "IDM_ADMIN_PASSWORD=$PASS" \ + -e "OCIS_FORCE_CONFIG_OVERWRITE=true" \ + -e "OCIS_URL=https://127.0.0.1:$PORT" \ + owncloud/ocis \ + init + + docker run --rm -d --name $NAME \ + -e "OCIS_LOG_LEVEL=debug" \ + -e "OCIS_LOG_PRETTY=true" \ + -e "OCIS_URL=https://127.0.0.1:$PORT" \ + -e "OCIS_ADMIN_USER_ID=some-admin-user-id-0000-100000000000" \ + -e "IDM_ADMIN_PASSWORD=$PASS" \ + -e "OCIS_INSECURE=true" \ + -e "PROXY_ENABLE_BASIC_AUTH=true" \ + -v ${CONF_DIR}:/etc/ocis \ + -p 127.0.0.1:${PORT}:9200 \ + owncloud/ocis + + echo type=webdav + echo url=https://127.0.0.1:${PORT}/dav/spaces/some-admin-user-id-0000-100000000000 + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo vendor=infinitescale + echo _connect=127.0.0.1:${PORT} + echo _connect_delay=5s +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestWebdavNextcloud b/fstest/testserver/init.d/TestWebdavNextcloud new file mode 100755 index 0000000..42766e5 --- /dev/null +++ b/fstest/testserver/init.d/TestWebdavNextcloud @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -e + +NAME=nextcloud +USER=rclone +PASS=ArmorAbleMale6 +PORT=28629 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "SQLITE_DATABASE=nextcloud.db" \ + -e "NEXTCLOUD_ADMIN_USER=rclone" \ + -e "NEXTCLOUD_ADMIN_PASSWORD=$PASS" \ + -e "NEXTCLOUD_TRUSTED_DOMAINS=*.*.*.*" \ + -p 127.0.0.1:${PORT}:80 \ + nextcloud:latest + + echo type=webdav + echo url=http://127.0.0.1:${PORT}/remote.php/dav/files/$USER/ + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo vendor=nextcloud + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestWebdavOwncloud b/fstest/testserver/init.d/TestWebdavOwncloud new file mode 100755 index 0000000..d2dc238 --- /dev/null +++ b/fstest/testserver/init.d/TestWebdavOwncloud @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -e + +NAME=owncloud +USER=rclone +PASS=HarperGrayerFewest5 +PORT=38081 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "OWNCLOUD_DOMAIN=localhost:8080" \ + -e "OWNCLOUD_DB_TYPE=sqlite" \ + -e "OWNCLOUD_DB_NAME=owncloud.db" \ + -e "OWNCLOUD_ADMIN_USERNAME=$USER" \ + -e "OWNCLOUD_ADMIN_PASSWORD=$PASS" \ + -e "OWNCLOUD_MYSQL_UTF8MB4=true" \ + -e "OWNCLOUD_REDIS_ENABLED=false" \ + -e "OWNCLOUD_TRUSTED_DOMAINS=127.0.0.1" \ + -p 127.0.0.1:${PORT}:8080 \ + owncloud/server + + echo type=webdav + echo url=http://127.0.0.1:${PORT}/remote.php/webdav/ + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo vendor=owncloud + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestWebdavRclone b/fstest/testserver/init.d/TestWebdavRclone new file mode 100755 index 0000000..e740ecc --- /dev/null +++ b/fstest/testserver/init.d/TestWebdavRclone @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +NAME=rclone-serve-webdav +USER=rclone +PASS=PagansSwimExpiry9 +IP=127.0.0.1 +PORT=28620 + +start() { + run rclone serve webdav --user $USER --pass $PASS --addr ${IP}:${PORT} ${DATADIR} + + echo type=webdav + echo vendor=rclone + echo url=http://${IP}:${PORT}/ + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo _connect=${IP}:$PORT +} + +. $(dirname "$0")/rclone-serve.bash diff --git a/fstest/testserver/init.d/docker.bash b/fstest/testserver/init.d/docker.bash new file mode 100644 index 0000000..1bcc2c2 --- /dev/null +++ b/fstest/testserver/init.d/docker.bash @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +stop() { + if status ; then + docker stop "$NAME" + echo "$NAME stopped" + fi +} + +status() { + if docker ps --format '{{.Names}}' | grep -q "^${NAME}$" ; then + echo "$NAME running" + else + echo "$NAME not running" + return 1 + fi + return 0 +} + +docker_ip() { + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{"\n"}}{{end}}' "$NAME" | head -n 1 +} diff --git a/fstest/testserver/init.d/rclone-serve.bash b/fstest/testserver/init.d/rclone-serve.bash new file mode 100644 index 0000000..408960e --- /dev/null +++ b/fstest/testserver/init.d/rclone-serve.bash @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# start an "rclone serve" server + +PIDFILE=/tmp/${NAME}.pid +DATADIR=/tmp/${NAME}-data + +stop() { + if status ; then + pid=$(cat "$PIDFILE") + kill "$pid" + rm "$PIDFILE" + echo "$NAME stopped" + fi +} + +status() { + if [ -e "$PIDFILE" ]; then + pid=$(cat "$PIDFILE") + if kill -0 "$pid" >/dev/null 2>&1; then + # echo "$NAME running" + return 0 + else + rm "$PIDFILE" + fi + fi + # echo "$NAME not running" + return 1 +} + +run() { + if ! status ; then + mkdir -p "$DATADIR" + nohup "$@" >> "/tmp/${NAME}.log" 2>&1 </dev/null & + pid=$! + echo $pid > "$PIDFILE" + disown "$pid" + fi +} + +# shellcheck disable=SC1090 +. "$(dirname "$0")/run.bash" diff --git a/fstest/testserver/init.d/run.bash b/fstest/testserver/init.d/run.bash new file mode 100644 index 0000000..ff1f70e --- /dev/null +++ b/fstest/testserver/init.d/run.bash @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail + +RUN_BASE="${STATE_DIR:-${XDG_RUNTIME_DIR:-/tmp}/rclone-test-server}" +: "${NAME:=$(basename "$0")}" +RUN_ROOT="${RUN_BASE}/${NAME}" +RUN_STATE="${RUN_ROOT}/state" +RUN_LOCK_FILE="${RUN_ROOT}/lock" +RUN_REF_COUNT="${RUN_STATE}/refcount" +RUN_OUTPUT="${RUN_STATE}/env" + +mkdir -p "${RUN_STATE}" +[[ -f "${RUN_REF_COUNT}" ]] || echo 0 >"${RUN_REF_COUNT}" +[[ -f "${RUN_OUTPUT}" ]] || : >"${RUN_OUTPUT}" +: > "${RUN_LOCK_FILE}" # ensure file exists + +# status helper that won't trip set -e +_is_running() { set +e; status >/dev/null 2>&1; local rc=$?; set -e; return $rc; } + +_acquire_lock() { + # open fd 9 on lock file and take exclusive lock + exec 9>"${RUN_LOCK_FILE}" + flock -x 9 +} + +_release_lock() { + flock -u 9 + exec 9>&- +} + +case "${1:-}" in + start) + _acquire_lock + trap '_release_lock' EXIT + + rc=$(cat "${RUN_REF_COUNT}" 2>/dev/null || echo 0) + + if (( rc == 0 )); then + # First client: ensure a clean instance, then start and cache env + if _is_running; then + stop || true + fi + if ! out="$(start)"; then + echo "failed to start" >&2 + exit 1 + fi + printf "%s\n" "$out" > "${RUN_OUTPUT}" + else + # Already owned: make sure it’s still up; if not, restart and refresh env + if ! _is_running; then + if ! out="$(start)"; then + echo "failed to restart" >&2 + exit 1 + fi + printf "%s\n" "$out" > "${RUN_OUTPUT}" + fi + fi + + rc=$((rc+1)); echo "${rc}" > "${RUN_REF_COUNT}" + cat "${RUN_OUTPUT}" + + trap - EXIT + _release_lock + ;; + + stop) + _acquire_lock + trap '_release_lock' EXIT + + rc=$(cat "${RUN_REF_COUNT}" 2>/dev/null || echo 0) + if (( rc > 0 )); then rc=$((rc-1)); fi + echo "${rc}" > "${RUN_REF_COUNT}" + if (( rc == 0 )) && _is_running; then + stop || true + fi + + trap - EXIT + _release_lock + ;; + + reset) + _acquire_lock + trap '_release_lock' EXIT + + stop || true + rm -rf "${RUN_BASE}" + + trap - EXIT + _release_lock + ;; + + status) + # passthrough; do NOT take the lock + status + ;; + + *) + echo "usage: $0 {start|stop|reset|status}" >&2 + exit 2 + ;; +esac diff --git a/fstest/testserver/init.d/seafile/docker-compose.yml b/fstest/testserver/init.d/seafile/docker-compose.yml new file mode 100644 index 0000000..8e0a099 --- /dev/null +++ b/fstest/testserver/init.d/seafile/docker-compose.yml @@ -0,0 +1,31 @@ +version: '2.0' +services: + db: + image: mariadb:10.5 + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_LOG_CONSOLE=true + volumes: + - ${SEAFILE_TEST_DATA}/${NAME}/seafile-mysql/db:/var/lib/mysql + + memcached: + image: memcached:1.6.9 + entrypoint: memcached -m 256 + + seafile: + image: seafileltd/seafile-mc:${SEAFILE_VERSION} + ports: + - "${SEAFILE_IP}:${SEAFILE_PORT}:80" + volumes: + - ${SEAFILE_TEST_DATA}/${NAME}/seafile-data:/shared + environment: + - DB_HOST=db + - DB_ROOT_PASSWD=${MYSQL_ROOT_PASSWORD} + - TIME_ZONE=Etc/UTC + - SEAFILE_ADMIN_EMAIL=${SEAFILE_ADMIN_EMAIL} + - SEAFILE_ADMIN_PASSWORD=${SEAFILE_ADMIN_PASSWORD} + - SEAFILE_SERVER_LETSENCRYPT=false + - SEAFILE_SERVER_HOSTNAME=${SEAFILE_IP}:${SEAFILE_PORT} + depends_on: + - db + - memcached diff --git a/fstest/testserver/testserver.go b/fstest/testserver/testserver.go new file mode 100644 index 0000000..9967bf5 --- /dev/null +++ b/fstest/testserver/testserver.go @@ -0,0 +1,198 @@ +// Package testserver starts and stops test servers if required +package testserver + +import ( + "bytes" + "errors" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/fspath" +) + +var ( + findConfigOnce sync.Once + configDir string // where the config is stored +) + +// Assume we are run somewhere within the rclone root +func findConfig() (string, error) { + dir := filepath.Join("fstest", "testserver", "init.d") + for range 5 { + fi, err := os.Stat(dir) + if err == nil && fi.IsDir() { + return filepath.Abs(dir) + } else if !os.IsNotExist(err) { + return "", err + } + dir = filepath.Join("..", dir) + } + return "", errors.New("couldn't find testserver config files - run from within rclone source") +} + +// returns path to a script to start this server +func cmdPath(name string) string { + return filepath.Join(configDir, name) +} + +// return true if the server with name has a start command +func hasStartCommand(name string) bool { + fi, err := os.Stat(cmdPath(name)) + return err == nil && !fi.IsDir() +} + +// run the command returning the output and an error +func run(name, command string) (out []byte, err error) { + script := cmdPath(name) + cmd := exec.Command(script, command) + out, err = cmd.CombinedOutput() + if err != nil { + err = fmt.Errorf("failed to run %s %s\n%s: %w", script, command, string(out), err) + } + return out, err +} + +// envKey returns the environment variable name to set name, key +func envKey(name, key string) string { + return fmt.Sprintf("RCLONE_CONFIG_%s_%s", strings.ToUpper(name), strings.ToUpper(key)) +} + +// match a line of config var=value +var matchLine = regexp.MustCompile(`^([a-zA-Z_]+)=(.*)$`) + +// Start the server and env vars so rclone can use it +func start(name string) error { + fs.Logf(name, "Starting server") + out, err := run(name, "start") + if err != nil { + return err + } + // parse the output and set environment vars from it + var connect string + var connectDelay time.Duration + for line := range bytes.SplitSeq(out, []byte("\n")) { + line = bytes.TrimSpace(line) + part := matchLine.FindSubmatch(line) + if part != nil { + key, value := part[1], part[2] + if string(key) == "_connect" { + connect = string(value) + continue + } else if string(key) == "_connect_delay" { + connectDelay, err = time.ParseDuration(string(value)) + if err != nil { + return fmt.Errorf("bad _connect_delay: %w", err) + } + continue + } + + // fs.Debugf(name, "key = %q, envKey = %q, value = %q", key, envKey(name, string(key)), value) + err = os.Setenv(envKey(name, string(key)), string(value)) + if err != nil { + return err + } + } + } + if connect == "" { + fs.Logf(name, "Started server") + return nil + } + // If we got a _connect value then try to connect to it + const maxTries = 100 + var rdBuf = make([]byte, 1) + for i := 1; i <= maxTries; i++ { + if i != 0 { + time.Sleep(time.Second) + } + fs.Logf(name, "Attempting to connect to %q try %d/%d", connect, i, maxTries) + conn, err := net.DialTimeout("tcp", connect, time.Second) + if err != nil { + fs.Debugf(name, "Connection to %q failed try %d/%d: %v", connect, i, maxTries, err) + continue + } + + err = conn.SetReadDeadline(time.Now().Add(time.Second)) + if err != nil { + return fmt.Errorf("failed to set deadline: %w", err) + } + n, err := conn.Read(rdBuf) + _ = conn.Close() + fs.Debugf(name, "Read %d, error: %v", n, err) + if err != nil && !errors.Is(err, os.ErrDeadlineExceeded) { + // Try again + continue + } + if connectDelay > 0 { + fs.Logf(name, "Connect delay %v", connectDelay) + time.Sleep(connectDelay) + } + fs.Logf(name, "Started server and connected to %q", connect) + return nil + } + return fmt.Errorf("failed to connect to %q on %q", name, connect) +} + +// Stops the named test server +func stop(name string) { + fs.Logf(name, "Stopping server") + _, err := run(name, "stop") + if err != nil { + fs.Errorf(name, "Failed to stop server: %v", err) + } +} + +// No server to stop so do nothing +func stopNothing() { +} + +// Start starts the test server for remoteName. +// +// This must be stopped by calling the function returned when finished. +func Start(remote string) (fn func(), err error) { + // don't start the local backend + if remote == "" { + return stopNothing, nil + } + parsed, err := fspath.Parse(remote) + if err != nil { + return nil, err + } + name := parsed.ConfigString + // don't start the local backend + if name == "" { + return stopNothing, nil + } + + // Make sure we know where the config is + findConfigOnce.Do(func() { + configDir, err = findConfig() + }) + if err != nil { + return nil, err + } + + // If remote has no start command then do nothing + if !hasStartCommand(name) { + return stopNothing, nil + } + + // Start the server + err = start(name) + if err != nil { + return nil, err + } + + // And return a function to stop it + return func() { + stop(name) + }, nil + +} diff --git a/fstest/testy/testy.go b/fstest/testy/testy.go new file mode 100644 index 0000000..5b73c01 --- /dev/null +++ b/fstest/testy/testy.go @@ -0,0 +1,20 @@ +// Package testy contains test utilities for rclone +package testy + +import ( + "os" + "testing" +) + +// CI returns true if we are running on the CI server +func CI() bool { + return os.Getenv("CI") != "" +} + +// SkipUnreliable skips this test if running on CI +func SkipUnreliable(t *testing.T) { + if !CI() { + return + } + t.Skip("Skipping Unreliable Test on CI") +} |