1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
|
// Config handling
// Package runs provides the types used by test_all
package runs
import (
"fmt"
"os"
"path"
"slices"
"github.com/rclone/rclone/fs"
yaml "gopkg.in/yaml.v3"
)
// Test describes an integration test to run with `go test`
type Test struct {
Path string // path to the source directory
FastList bool // if it is possible to add -fast-list to tests
Short bool // if it is possible to run the test with -short
AddBackend bool // set if Path needs the current backend appending
NoRetries bool // set if no retries should be performed
NoBinary bool // set to not build a binary in advance
LocalOnly bool // if set only run with the local backend
}
// Backend describes a backend test
//
// FIXME make bucket-based remotes set sub-dir automatically???
type Backend struct {
Backend string // name of the backend directory
Remote string // name of the test remote
FastList bool // set to test with -fast-list
Short bool // set to test with -short
OneOnly bool // set to run only one backend test at once
MaxFile string // file size limit
CleanUp bool // when running clean, run cleanup first
Ignore []string // test names to ignore the failure of
Tests []string // paths of tests to run, blank for all
IgnoreTests []string // paths of tests not to run, blank for none
ListRetries int // -list-retries if > 0
ExtraTime float64 // factor to multiply the timeout by
Env []string // environment variables to set in form KEY=VALUE
}
// includeTest returns true if this backend should be included in this
// test
func (b *Backend) includeTest(t *Test) bool {
// Is this test ignored
if slices.Contains(b.IgnoreTests, t.Path) {
return false
}
// Empty b.Tests imples do all of them except the ignored
if len(b.Tests) == 0 {
return true
}
return slices.Contains(b.Tests, t.Path)
}
// MakeRuns creates Run objects the Backend and Test
//
// There can be several created, one for each combination of optional
// flags (e.g. FastList)
func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
if !b.includeTest(t) {
return runs
}
maxSize := fs.SizeSuffix(0)
if b.MaxFile != "" {
if err := maxSize.Set(b.MaxFile); err != nil {
fs.Logf(nil, "Invalid maxfile value %q: %v", b.MaxFile, err)
}
}
fastlists := []bool{false}
if b.FastList && t.FastList {
fastlists = append(fastlists, true)
}
ignore := make(map[string]struct{}, len(b.Ignore))
for _, item := range b.Ignore {
ignore[item] = struct{}{}
}
for _, fastlist := range fastlists {
if t.LocalOnly && b.Backend != "local" {
continue
}
run := &Run{
Remote: b.Remote,
Backend: b.Backend,
Path: t.Path,
FastList: fastlist,
Short: (b.Short && t.Short),
NoRetries: t.NoRetries,
OneOnly: b.OneOnly,
NoBinary: t.NoBinary,
SizeLimit: int64(maxSize),
Ignore: ignore,
ListRetries: b.ListRetries,
ExtraTime: b.ExtraTime,
Env: b.Env,
}
if t.AddBackend {
run.Path = path.Join(run.Path, b.Backend)
}
runs = append(runs, run)
}
return runs
}
// Config describes the config for this program
type Config struct {
Tests []Test
Backends []Backend
}
// NewConfig reads the config file
func NewConfig(configFile string) (*Config, error) {
d, err := os.ReadFile(configFile)
if err != nil {
return nil, fmt.Errorf("failed to read config file: %w", err)
}
config := &Config{}
err = yaml.Unmarshal(d, &config)
if err != nil {
return nil, fmt.Errorf("failed to parse config file: %w", err)
}
// d, err = yaml.Marshal(&config)
// if err != nil {
// log.Fatalf("error: %v", err)
// }
// fmt.Printf("--- m dump:\n%s\n\n", string(d))
return config, nil
}
// MakeRuns makes Run objects for each combination of Backend and Test
// in the config
func (c *Config) MakeRuns() (runs Runs) {
for _, backend := range c.Backends {
for _, test := range c.Tests {
runs = append(runs, backend.MakeRuns(&test)...)
}
}
return runs
}
// FilterBackendsByRemotes filters the Backends with the remotes passed in.
//
// If no backend is found with a remote is found then synthesize one
func (c *Config) FilterBackendsByRemotes(remotes []string) {
var newBackends []Backend
for _, name := range remotes {
found := false
for i := range c.Backends {
if c.Backends[i].Remote == name {
newBackends = append(newBackends, c.Backends[i])
found = true
}
}
if !found {
fs.Logf(nil, "Remote %q not found - inserting with default flags", name)
// Lookup which backend
fsInfo, _, _, _, err := fs.ConfigFs(name)
if err != nil {
fs.Fatalf(nil, "couldn't find remote %q: %v", name, err)
}
newBackends = append(newBackends, Backend{Backend: fsInfo.FileName(), Remote: name})
}
}
c.Backends = newBackends
}
// FilterBackendsByBackends filters the Backends with the backendNames passed in
func (c *Config) FilterBackendsByBackends(backendNames []string) {
var newBackends []Backend
for _, name := range backendNames {
for i := range c.Backends {
if c.Backends[i].Backend == name {
newBackends = append(newBackends, c.Backends[i])
}
}
}
c.Backends = newBackends
}
// FilterTests filters the incoming tests into the backends selected
func (c *Config) FilterTests(paths []string) {
var newTests []Test
for _, path := range paths {
for i := range c.Tests {
if c.Tests[i].Path == path {
newTests = append(newTests, c.Tests[i])
}
}
}
c.Tests = newTests
}
|
