diff options
| author | Michael Tews <michael@tews.dev> | 2026-03-06 10:58:19 +0100 |
|---|---|---|
| committer | Michael Tews <michael@tews.dev> | 2026-03-12 15:23:26 +0100 |
| commit | ce94f2d69a5f1aab1fc8fc2947f0a6cfd81bb4d1 (patch) | |
| tree | 4ae72baff8af7a73d46ccd544463aac3f638df39 /fstest/testserver | |
| parent | a59763b4ff8c5728401232a696dfc8a725cf4e02 (diff) | |
test: adds fstest from rclone
Signed-off-by: Michael Tews <michael@tews.dev>
Diffstat (limited to 'fstest/testserver')
47 files changed, 1783 insertions, 0 deletions
diff --git a/fstest/testserver/images/test-hdfs/Dockerfile b/fstest/testserver/images/test-hdfs/Dockerfile new file mode 100644 index 0000000..07c621d --- /dev/null +++ b/fstest/testserver/images/test-hdfs/Dockerfile @@ -0,0 +1,45 @@ +# A very minimal hdfs server for integration testing rclone +FROM debian:stretch + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openjdk-8-jdk \ + net-tools curl python krb5-user krb5-kdc krb5-admin-server \ + && rm -rf /var/lib/apt/lists/* + +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ + +ENV HADOOP_VERSION 3.2.1 +ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz +RUN set -x \ + && curl -fSL "$HADOOP_URL" -o /tmp/hadoop.tar.gz \ + && tar -xvf /tmp/hadoop.tar.gz -C /opt/ \ + && rm /tmp/hadoop.tar.gz* + +RUN ln -s /opt/hadoop-$HADOOP_VERSION/etc/hadoop /etc/hadoop +RUN mkdir /opt/hadoop-$HADOOP_VERSION/logs + +RUN mkdir /hadoop-data +RUN mkdir -p /hadoop/dfs/name +RUN mkdir -p /hadoop/dfs/data + +ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION +ENV HADOOP_CONF_DIR=/etc/hadoop +ENV MULTIHOMED_NETWORK=1 + +ENV USER=root +ENV PATH $HADOOP_HOME/bin/:$PATH + +ADD core-site.xml /etc/hadoop/core-site.xml +ADD hdfs-site.xml /etc/hadoop/hdfs-site.xml +ADD httpfs-site.xml /etc/hadoop/httpfs-site.xml +ADD kms-site.xml /etc/hadoop/kms-site.xml +ADD mapred-site.xml /etc/hadoop/mapred-site.xml +ADD yarn-site.xml /etc/hadoop/yarn-site.xml + +ADD krb5.conf /etc/ +ADD kdc.conf /etc/krb5kdc/ +RUN echo '*/admin@KERBEROS.RCLONE *' > /etc/krb5kdc/kadm5.acl + +ADD run.sh /run.sh +RUN chmod a+x /run.sh +CMD ["/run.sh"] diff --git a/fstest/testserver/images/test-hdfs/README.md b/fstest/testserver/images/test-hdfs/README.md new file mode 100644 index 0000000..1f760dd --- /dev/null +++ b/fstest/testserver/images/test-hdfs/README.md @@ -0,0 +1,57 @@ +# Test HDFS + +This is a docker image for rclone's integration tests which runs an +hdfs filesystem in a docker image. + +## Build + +``` +docker build --rm -t rclone/test-hdfs . +docker push rclone/test-hdfs +``` + +# Test + +configure remote: +``` +[TestHdfs] +type = hdfs +namenode = 127.0.0.1:8020 +username = root +``` + +run tests +``` +cd backend/hdfs +GO111MODULE=on go test -v +``` + +hdfs logs will be available in `.stdout.log` and `.stderr.log` + +# Kerberos + +test can be run against kerberos-enabled hdfs + +1. configure local krb5.conf + ``` + [libdefaults] + default_realm = KERBEROS.RCLONE + [realms] + KERBEROS.RCLONE = { + kdc = localhost + } + ``` + +2. enable kerberos in remote configuration + ``` + [TestHdfs] + ... + service_principal_name = hdfs/localhost + data_transfer_protection = privacy + ``` + +3. run test + ``` + cd backend/hdfs + KERBEROS=true GO111MODULE=on go test -v + ```
\ No newline at end of file diff --git a/fstest/testserver/images/test-hdfs/core-site.xml b/fstest/testserver/images/test-hdfs/core-site.xml new file mode 100644 index 0000000..061d48d --- /dev/null +++ b/fstest/testserver/images/test-hdfs/core-site.xml @@ -0,0 +1,12 @@ +<configuration> + <property><name>fs.defaultFS</name><value>hdfs://localhost:8020</value></property> + <property><name>hadoop.http.staticuser.user</name><value>root</value></property> + <property><name>hadoop.proxyuser.root.groups</name><value>root,nogroup</value></property> + <property><name>hadoop.proxyuser.root.hosts</name><value>*</value></property> + <!-- KERBEROS BEGIN --> + <property><name>hadoop.security.authentication</name><value>kerberos</value></property> + <property><name>hadoop.security.authorization</name><value>true</value></property> + <property><name>hadoop.rpc.protection</name><value>integrity</value></property> + <property><name>hadoop.user.group.static.mapping.overrides</name><value>user=supergroup</value></property> + <!-- KERBEROS END --> +</configuration> diff --git a/fstest/testserver/images/test-hdfs/hdfs-site.xml b/fstest/testserver/images/test-hdfs/hdfs-site.xml new file mode 100644 index 0000000..3f3f3a6 --- /dev/null +++ b/fstest/testserver/images/test-hdfs/hdfs-site.xml @@ -0,0 +1,31 @@ +<configuration> + <property><name>dfs.client.use.datanode.hostname</name><value>true</value></property> + <property><name>dfs.datanode.data.dir</name><value>file:///hadoop/dfs/data</value></property> + <property><name>dfs.datanode.use.datanode.hostname</name><value>true</value></property> + <property><name>dfs.namenode.accesstime.precision</name><value>3600000</value></property> + <property><name>dfs.namenode.http-bind-host</name><value>0.0.0.0</value></property> + <property><name>dfs.namenode.https-bind-host</name><value>0.0.0.0</value></property> + <property><name>dfs.namenode.name.dir</name><value>file:///hadoop/dfs/name</value></property> + <property><name>dfs.namenode.rpc-bind-host</name><value>0.0.0.0</value></property> + <property><name>dfs.namenode.safemode.extension</name><value>5000</value></property> + <property><name>dfs.namenode.servicerpc-bind-host</name><value>0.0.0.0</value></property> + <property><name>dfs.replication</name><value>2</value></property> + <property><name>nfs.dump.dir</name><value>/tmp</value></property> + <!-- KERBEROS BEGIN --> + <property><name>ignore.secure.ports.for.testing</name><value>true</value></property> + <property><name>dfs.safemode.extension</name><value>0</value></property> + <property><name>dfs.block.access.token.enable</name><value>true</value></property> + + <property><name>dfs.encrypt.data.transfer</name><value>true</value></property> + <property><name>dfs.encrypt.data.transfer.algorithm</name><value>rc4</value></property> + <property><name>dfs.encrypt.data.transfer.cipher.suites</name><value>AES/CTR/NoPadding</value></property> + + <property><name>dfs.namenode.kerberos.principal</name> <value>hdfs/_HOST@KERBEROS.RCLONE</value></property> + <property><name>dfs.web.authentication.kerberos.principal</name><value>HTTP/_HOST@KERBEROS.RCLONE</value></property> + <property><name>dfs.datanode.kerberos.principal</name> <value>hdfs/_HOST@KERBEROS.RCLONE</value></property> + + <property><name>dfs.namenode.keytab.file</name> <value>/etc/hadoop/kerberos.key</value></property> + <property><name>dfs.web.authentication.kerberos.keytab</name><value>/etc/hadoop/kerberos.key</value></property> + <property><name>dfs.datanode.keytab.file</name> <value>/etc/hadoop/kerberos.key</value></property> + <!-- KERBEROS END --> +</configuration> diff --git a/fstest/testserver/images/test-hdfs/httpfs-site.xml b/fstest/testserver/images/test-hdfs/httpfs-site.xml new file mode 100644 index 0000000..8313843 --- /dev/null +++ b/fstest/testserver/images/test-hdfs/httpfs-site.xml @@ -0,0 +1,2 @@ +<configuration> +</configuration> diff --git a/fstest/testserver/images/test-hdfs/kdc.conf b/fstest/testserver/images/test-hdfs/kdc.conf new file mode 100644 index 0000000..9eeb0bb --- /dev/null +++ b/fstest/testserver/images/test-hdfs/kdc.conf @@ -0,0 +1,4 @@ +[realms] + KERBEROS.RCLONE = { + acl_file = /etc/krb5kdc/kadm5.acl + } diff --git a/fstest/testserver/images/test-hdfs/kms-site.xml b/fstest/testserver/images/test-hdfs/kms-site.xml new file mode 100644 index 0000000..8313843 --- /dev/null +++ b/fstest/testserver/images/test-hdfs/kms-site.xml @@ -0,0 +1,2 @@ +<configuration> +</configuration> diff --git a/fstest/testserver/images/test-hdfs/krb5.conf b/fstest/testserver/images/test-hdfs/krb5.conf new file mode 100644 index 0000000..012950b --- /dev/null +++ b/fstest/testserver/images/test-hdfs/krb5.conf @@ -0,0 +1,10 @@ +[libdefaults] + default_realm = KERBEROS.RCLONE + dns_lookup_realm = false + dns_lookup_kdc = false + forwardable = true + proxiable = true +[realms] + KERBEROS.RCLONE = { + kdc = localhost + } diff --git a/fstest/testserver/images/test-hdfs/mapred-site.xml b/fstest/testserver/images/test-hdfs/mapred-site.xml new file mode 100644 index 0000000..9f70286 --- /dev/null +++ b/fstest/testserver/images/test-hdfs/mapred-site.xml @@ -0,0 +1,5 @@ +<configuration> + <property><name>mapreduce.framework.name</name><value>yarn</value></property> + <property><name>yarn.nodemanager.bind-host</name><value>0.0.0.0</value></property> +</configuration> + diff --git a/fstest/testserver/images/test-hdfs/run.sh b/fstest/testserver/images/test-hdfs/run.sh new file mode 100755 index 0000000..207e11a --- /dev/null +++ b/fstest/testserver/images/test-hdfs/run.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +KERBEROS=${KERBEROS-"false"} + +if [ $KERBEROS = "true" ]; then + echo prepare kerberos + ADMIN_PASSWORD="kerberos" + USER_PASSWORD="user" + + echo -e "$ADMIN_PASSWORD\n$ADMIN_PASSWORD" | kdb5_util -r "KERBEROS.RCLONE" create -s + echo -e "$ADMIN_PASSWORD\n$ADMIN_PASSWORD" | kadmin.local -q "addprinc hadoop/admin" + echo -e "$USER_PASSWORD\n$USER_PASSWORD" | kadmin.local -q "addprinc user" + kadmin.local -q 'addprinc -randkey hdfs/localhost' + kadmin.local -q 'addprinc -randkey hdfs/rclone-hdfs' + kadmin.local -q 'addprinc -randkey HTTP/localhost' + kadmin.local -p hadoop/admin -q "ktadd -k /etc/hadoop/kerberos.key hdfs/localhost hdfs/rclone-hdfs HTTP/localhost" + service krb5-kdc restart + echo -e "$USER_PASSWORD\n" | kinit user + klist + echo kerberos ready +else + echo drop kerberos from configuration files + sed -i '/KERBEROS BEGIN/,/KERBEROS END/d' /etc/hadoop/core-site.xml + sed -i '/KERBEROS BEGIN/,/KERBEROS END/d' /etc/hadoop/hdfs-site.xml +fi + + +echo format namenode +hdfs namenode -format test + +hdfs namenode & +hdfs datanode & +exec sleep infinity diff --git a/fstest/testserver/images/test-hdfs/yarn-site.xml b/fstest/testserver/images/test-hdfs/yarn-site.xml new file mode 100644 index 0000000..ade8c7f --- /dev/null +++ b/fstest/testserver/images/test-hdfs/yarn-site.xml @@ -0,0 +1,14 @@ +<configuration> + <property><name>yarn.log-aggregation-enable</name><value>true</value></property> + <property><name>yarn.log.server.url</name><value>http://localhost:8188/applicationhistory/logs/</value></property> + <property><name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name><value>org.apache.hadoop.mapred.ShuffleHandler</value></property> + <property><name>yarn.nodemanager.aux-services</name><value>mapreduce_shuffle</value></property> + <property><name>yarn.nodemanager.bind-host</name><value>0.0.0.0</value></property> + <property><name>yarn.nodemanager.bind-host</name><value>0.0.0.0</value></property> + <property><name>yarn.nodemanager.remote-app-log-dir</name><value>/app-logs</value></property> + <property><name>yarn.timeline-service.bind-host</name><value>0.0.0.0</value></property> + <property><name>yarn.timeline-service.enabled</name><value>true</value></property> + <property><name>yarn.timeline-service.generic-application-history.enabled</name><value>true</value></property> + <property><name>yarn.timeline-service.hostname</name><value>historyserver.hadoop</value></property> + <property><name>yarn.timeline-service.leveldb-timeline-store.path</name><value>/hadoop/yarn/timeline</value></property> +</configuration> diff --git a/fstest/testserver/images/test-sftp-openssh/Dockerfile b/fstest/testserver/images/test-sftp-openssh/Dockerfile new file mode 100644 index 0000000..f7a8455 --- /dev/null +++ b/fstest/testserver/images/test-sftp-openssh/Dockerfile @@ -0,0 +1,11 @@ +# A very minimal sftp server for integration testing rclone +FROM alpine:latest + +# User rclone, password password +RUN \ + apk add openssh && \ + ssh-keygen -A && \ + adduser -D rclone && \ + echo "rclone:password" | chpasswd + +ENTRYPOINT [ "/usr/sbin/sshd", "-D" ] diff --git a/fstest/testserver/images/test-sftp-openssh/README.md b/fstest/testserver/images/test-sftp-openssh/README.md new file mode 100644 index 0000000..2e98b44 --- /dev/null +++ b/fstest/testserver/images/test-sftp-openssh/README.md @@ -0,0 +1,17 @@ +# Test SFTP Openssh + +This is a docker image for rclone's integration tests which runs an +openssh server in a docker image. + +## Build + +``` +docker build --rm -t rclone/test-sftp-openssh . +docker push rclone/test-sftp-openssh +``` + +# Test + +``` +rclone lsf -R --sftp-host 172.17.0.2 --sftp-user rclone --sftp-pass $(rclone obscure password) :sftp: +``` diff --git a/fstest/testserver/init.d/PORTS.md b/fstest/testserver/init.d/PORTS.md new file mode 100644 index 0000000..fc955a2 --- /dev/null +++ b/fstest/testserver/init.d/PORTS.md @@ -0,0 +1,49 @@ +# Ports for tests + +All these tests need to run on a different port. + +They should be bound to localhost so they are not accessible externally. + +| Port | Test | +|:-----:|:----:| +| 88 | TestHdfs | +| 750 | TestHdfs | +| 8020 | TestHdfs | +| 8086 | TestSeafileV6 | +| 8087 | TestSeafile | +| 8088 | TestSeafileEncrypted | +| 9866 | TestHdfs | +| 28620 | TestWebdavRclone | +| 28621 | TestSFTPRclone | +| 28622 | TestFTPRclone | +| 28623 | TestSFTPRcloneSSH | +| 28624 | TestS3Rclone | +| 28625 | TestS3Minio | +| 28626 | TestS3MinioEdge | +| 28627 | TestSFTPOpenssh | +| 28628 | TestSwiftAIO | +| 28629 | TestWebdavNextcloud | +| 28630 | TestSMB | +| 28631 | TestFTPProftpd | +| 28632 | TestSwiftAIOsegments | +| 28633 | TestSMBKerberos | +| 28634 | TestSMBKerberos | +| 28635 | TestS3Exaba | +| 28636 | TestS3Exaba | +| 28637 | TestSMBKerberosCcache | +| 28638 | TestSMBKerberosCcache | +| 28639 | TestWebdavInfiniteScale | +| 38081 | TestWebdavOwncloud | + +## Non localhost tests + +All these use `$(docker_ip)` which means they don't work on macOS or +Windows. It is proabably possible to make them work with some effort +but will require port forwarding a range of ports and configuring the +FTP server to only use that range of ports. The FTP server will likely +need know it is behind a NAT so it advertises the correct external IP. + +- TestFTPProftpd +- TestFTPPureftpd +- TestFTPVsftpd +- TestFTPVsftpdTLS diff --git a/fstest/testserver/init.d/README.md b/fstest/testserver/init.d/README.md new file mode 100644 index 0000000..c9acd92 --- /dev/null +++ b/fstest/testserver/init.d/README.md @@ -0,0 +1,48 @@ +This directory contains scripts to start and stop servers for testing. + +The commands are named after the remotes in use. They are executable +files with the following parameters: + + start - starts the server if not running + stop - stops the server if nothing is using it + status - returns non-zero exit code if the server is not running + reset - stops the server and resets any reference counts + +These will be called automatically by test_all if that remote is +required. + +When start is run it should output config parameters for that remote. +If a `_connect` parameter is output then that will be used for a +connection test. For example if `_connect=127.0.0.1:80` then a TCP +connection will be made to `127.0.0.1:80` and only when that succeeds +will the test continue. + +If in addition to `_connect`, `_connect_delay=5s` is also present then +after the connection succeeds rclone will wait `5s` before continuing. +This is for servers that aren't quite ready even though they have +opened their TCP ports. + +## Writing new scripts + +A docker based server or an `rclone serve` based server should be easy +to write. Look at one of the examples. + +`run.bash` contains boilerplate to be included in a bash script for +interpreting the command line parameters. This does reference counting +to ensure multiple copies of the server aren't running at once. +Including this is mandatory. It will call your `start()`, `stop()` and +`status()` functions. + +`docker.bash` contains library functions to help with docker +implementations. It contains implementations of `stop()` and +`status()` so all you have to do is write a `start()` function. + +`rclone-serve.bash` contains functions to help with `rclone serve` +based implementations. It contains implementations of `stop()` and +`status()` so all you have to do is write a `start()` function which +should call the `run()` function provided. + +Any external TCP or UDP ports used should be unique as any of the +servers might be running together. So please create a new line in the +[PORTS](PORTS.md) file to allocate your server a port. Bind any ports +to localhost so they aren't accessible externally. diff --git a/fstest/testserver/init.d/TestFTPProftpd b/fstest/testserver/init.d/TestFTPProftpd new file mode 100755 index 0000000..029909a --- /dev/null +++ b/fstest/testserver/init.d/TestFTPProftpd @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -e + +NAME=proftpd +USER=rclone +PASS=RaidedBannedPokes5 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "FTP_USERNAME=rclone" \ + -e "FTP_PASSWORD=$PASS" \ + hauptmedia/proftpd + + echo type=ftp + echo host=$(docker_ip) + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo encoding=Asterisk,Ctl,Dot,Slash + echo _connect=$(docker_ip):21 +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestFTPPureftpd b/fstest/testserver/init.d/TestFTPPureftpd new file mode 100755 index 0000000..69c9285 --- /dev/null +++ b/fstest/testserver/init.d/TestFTPPureftpd @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -e + +NAME=pureftpd +USER=rclone +PASS=AcridSpiesBooks2 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "FTP_USER_NAME=rclone" \ + -e "FTP_USER_PASS=$PASS" \ + -e "FTP_USER_HOME=/data" \ + -e "FTP_MAX_CLIENTS=50" \ + -e "FTP_MAX_CONNECTIONS=50" \ + -e "FTP_PASSIVE_PORTS=30000:40000" \ + stilliard/pure-ftpd + + echo type=ftp + echo host=$(docker_ip) + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo encoding=BackSlash,Ctl,Del,Dot,RightSpace,Slash,SquareBracket + echo _connect=$(docker_ip):21 +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestFTPRclone b/fstest/testserver/init.d/TestFTPRclone new file mode 100755 index 0000000..85ad26a --- /dev/null +++ b/fstest/testserver/init.d/TestFTPRclone @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +NAME=rclone-serve-ftp +USER=rclone +PASS=FuddleIdlingJell5 +IP=127.0.0.1 +PORT=28622 + +start() { + run rclone serve ftp --user $USER --pass $PASS --addr ${IP}:${PORT} ${DATADIR} + + echo type=ftp + echo host=${IP} + echo port=$PORT + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo _connect=${IP}:${PORT} +} + +. $(dirname "$0")/rclone-serve.bash diff --git a/fstest/testserver/init.d/TestFTPVsftpd b/fstest/testserver/init.d/TestFTPVsftpd new file mode 100755 index 0000000..d33dcf0 --- /dev/null +++ b/fstest/testserver/init.d/TestFTPVsftpd @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -e + +NAME=vsftpd +USER=rclone +PASS=TiffedRestedSian4 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "FTP_USER=rclone" \ + -e "FTP_PASS=$PASS" \ + fauria/vsftpd + + echo type=ftp + echo host=$(docker_ip) + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo writing_mdtm=true + echo encoding=Ctl,LeftPeriod,Slash + echo _connect=$(docker_ip):21 +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestFTPVsftpdTLS b/fstest/testserver/init.d/TestFTPVsftpdTLS new file mode 100755 index 0000000..ebcd3b0 --- /dev/null +++ b/fstest/testserver/init.d/TestFTPVsftpdTLS @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -e + +NAME=vsftpdtls +USER=rclone +PASS=TiffedRestedSian4 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "FTP_USER=rclone" \ + -e "FTP_PASS=$PASS" \ + rclone/vsftpd + + echo type=ftp + echo host=$(docker_ip) + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo writing_mdtm=true + echo encoding=Ctl,LeftPeriod,Slash + echo _connect=$(docker_ip):21 +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestHdfs b/fstest/testserver/init.d/TestHdfs new file mode 100755 index 0000000..26aea1a --- /dev/null +++ b/fstest/testserver/init.d/TestHdfs @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -e + +NAME=rclone-hdfs +KERBEROS=${KERBEROS-"false"} + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name "rclone-hdfs" \ + --hostname "rclone-hdfs" \ + -e "KERBEROS=$KERBEROS" \ + -p 127.0.0.1:9866:9866 \ + -p 127.0.0.1:8020:8020 \ + -p 127.0.0.1:750:750 \ + -p 127.0.0.1:88:88 \ + rclone/test-hdfs + sleep 30 + + if [ $KERBEROS = "true" ]; then + docker cp rclone-hdfs:/tmp/krb5cc_0 /tmp/krb5cc_`id -u` + fi + + echo type=hdfs + echo namenode=127.0.0.1:8020 + echo username=root + echo _connect=127.0.0.1:8020 +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestS3Exaba b/fstest/testserver/init.d/TestS3Exaba new file mode 100755 index 0000000..c9a7f92 --- /dev/null +++ b/fstest/testserver/init.d/TestS3Exaba @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -e + +NAME=exaba +USER="Use the webui to find the access_key_id" +PASS="Use the webui to find the secret_access_key" +PORT=28635 +WEBUIPORT=28636 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "CLUSTER_NAME=rclone" \ + -e "CLUSTER_SIZE_GB=20" \ + -p 127.0.0.1:${PORT}:9000 \ + -p 127.0.0.1:${WEBUIPORT}:9006 \ + exaba/exaba + + echo type=s3 + echo provider=Exaba + echo access_key_id=$USER + echo secret_access_key=$PASS + echo endpoint=http://127.0.0.1:${PORT}/ + echo webui=http://127.0.0.1:${WEBUIPORT}/ + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestS3Minio b/fstest/testserver/init.d/TestS3Minio new file mode 100755 index 0000000..b4d3dde --- /dev/null +++ b/fstest/testserver/init.d/TestS3Minio @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -e + +NAME=minio +USER=rclone +PASS=AxedBodedGinger7 +PORT=28625 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "MINIO_ACCESS_KEY=$USER" \ + -e "MINIO_SECRET_KEY=$PASS" \ + -p 127.0.0.1:${PORT}:9000 \ + minio/minio server /data + + echo type=s3 + echo provider=Minio + echo access_key_id=$USER + echo secret_access_key=$PASS + echo endpoint=http://127.0.0.1:${PORT}/ + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestS3MinioEdge b/fstest/testserver/init.d/TestS3MinioEdge new file mode 100755 index 0000000..399ec7f --- /dev/null +++ b/fstest/testserver/init.d/TestS3MinioEdge @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -e + +NAME=minio-edge +USER=rclone +PASS=DeniseOxygenEiffel4 +PORT=28626 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "MINIO_ACCESS_KEY=$USER" \ + -e "MINIO_SECRET_KEY=$PASS" \ + -p 127.0.0.1:${PORT}:9000 \ + minio/minio:edge server /data + + echo type=s3 + echo provider=Minio + echo access_key_id=$USER + echo secret_access_key=$PASS + echo endpoint=http://127.0.0.1:${PORT}/ + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestS3Rclone b/fstest/testserver/init.d/TestS3Rclone new file mode 100755 index 0000000..c336322 --- /dev/null +++ b/fstest/testserver/init.d/TestS3Rclone @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +NAME=rclone-serve-s3 +ACCESS_KEY_ID=rclone +SECRET_ACCESS_KEY=JoltRogueVerde5 +IP=127.0.0.1 +PORT=28624 + +start() { + run rclone serve s3 --auth-key ${ACCESS_KEY_ID},${SECRET_ACCESS_KEY} --addr ${IP}:${PORT} ${DATADIR} + + echo type=s3 + echo provider=Rclone + echo endpoint=http://${IP}:${PORT}/ + echo access_key_id=${ACCESS_KEY_ID} + echo secret_access_key=${SECRET_ACCESS_KEY} + echo _connect=${IP}:${PORT} +} + +. $(dirname "$0")/rclone-serve.bash diff --git a/fstest/testserver/init.d/TestSFTPOpenssh b/fstest/testserver/init.d/TestSFTPOpenssh new file mode 100755 index 0000000..91a9c9a --- /dev/null +++ b/fstest/testserver/init.d/TestSFTPOpenssh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -e + +NAME=rclone-sftp-openssh +USER=rclone +PASS=password +PORT=28627 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name ${NAME} \ + -p 127.0.0.1:${PORT}:22 \ + rclone/test-sftp-openssh + + echo type=sftp + echo host=127.0.0.1 + echo port=$PORT + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo copy_is_hardlink=true + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSFTPRclone b/fstest/testserver/init.d/TestSFTPRclone new file mode 100755 index 0000000..f553112 --- /dev/null +++ b/fstest/testserver/init.d/TestSFTPRclone @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +NAME=rclone-serve-sftp +USER=rclone +PASS=CranesBallotDorsey5 +IP=127.0.0.1 +PORT=28621 + +start() { + run rclone serve sftp --user $USER --pass $PASS --addr ${IP}:${PORT} ${DATADIR} + + echo type=sftp + echo host=${IP} + echo port=$PORT + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo _connect=${IP}:${PORT} +} + +. $(dirname "$0")/rclone-serve.bash diff --git a/fstest/testserver/init.d/TestSFTPRcloneSSH b/fstest/testserver/init.d/TestSFTPRcloneSSH new file mode 100755 index 0000000..989a5e6 --- /dev/null +++ b/fstest/testserver/init.d/TestSFTPRcloneSSH @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -e + +NAME=rclone-serve-sftp-ssh +IP=127.0.0.1 +PORT=28623 +PRIVATE_KEY=/tmp/${NAME}.key +PUBLIC_KEY=/tmp/${NAME}.pub + +start() { + +cat >${PRIVATE_KEY} <<'#EOF' +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEAv7e6d+AbxELvRk7sZipketuqgE4/vVbf/6PMuOd1OSPyOOsAOs41 +tvc4Sk4S+/6ReHW4l1DKy5IH0smOsA1k58kKFkN1NChHU5z0CitAiZwdl7zqvxNJqlMmYi +GTubhQdMnDrq0AAnhyr9TFrcZmYPcp9tHcpt9VQWeLYR/16tT53WnpgTuMWlgyM58bpCh/ +cDO7gOjSyXHhPPxrU1qdr5g/5T9HFgQfi2CX8vk4pDY+Qw1Lnp1MMpKT4i9xWGMU8oDJG3 +08RrzUi9tz1RTePtbs4xXOy8cXOZaAODDQok4iWvJEpGgJYLjhNuHzZiUDcfc1SkXvONui +7j5RC/rsQOYB5Sd7ATlF4HymAxZJ3iPu+eYBZi7lwIPeug+42WlVon+D5dOYmrgcPpAZv7 +67Lthv62FMmvc1SHHGPZLS3dWfbZeXayve9+wIkKFEuDN76zYAavjSRm9fBKny6J+noJgp +bDMVNnTfNA28fsNbsCS6OsBjLbiFjMHxhuYACMaVAAAFgBfF8CkXxfApAAAAB3NzaC1yc2 +EAAAGBAL+3unfgG8RC70ZO7GYqZHrbqoBOP71W3/+jzLjndTkj8jjrADrONbb3OEpOEvv+ +kXh1uJdQysuSB9LJjrANZOfJChZDdTQoR1Oc9AorQImcHZe86r8TSapTJmIhk7m4UHTJw6 +6tAAJ4cq/Uxa3GZmD3KfbR3KbfVUFni2Ef9erU+d1p6YE7jFpYMjOfG6Qof3Azu4Do0slx +4Tz8a1Nana+YP+U/RxYEH4tgl/L5OKQ2PkMNS56dTDKSk+IvcVhjFPKAyRt9PEa81Ivbc9 +UU3j7W7OMVzsvHFzmWgDgw0KJOIlryRKRoCWC44Tbh82YlA3H3NUpF7zjbou4+UQv67EDm +AeUnewE5ReB8pgMWSd4j7vnmAWYu5cCD3roPuNlpVaJ/g+XTmJq4HD6QGb++uy7Yb+thTJ +r3NUhxxj2S0t3Vn22Xl2sr3vfsCJChRLgze+s2AGr40kZvXwSp8uifp6CYKWwzFTZ03zQN +vH7DW7AkujrAYy24hYzB8YbmAAjGlQAAAAMBAAEAAAGABOxf8oIj1Gdvo5uVQI5oJCuN9l +uMEX2wpOz87earwPrmVoXabKgtAvTYUjgtDqGb9L75LZGak529a7FXY7gEVlt4UdgLo3pB +UqleLwCrWJ1UuTfVw3BoXOJjwvNfys4r6sPfrZWtwWJ8d318UhkdOfI+9qKvCu4DT3msP6 +NFenFbtU7p+zKfSRaou2CjohSUKTp63zWbbCbrhNhqnSpfkEnVojp8xdj3QmoJnOi/hqAJ ++0jVH06kzUusVounWoC41pTr1Dlnvy+gWhJcZtHNBXixL6JCM9XGh+z0XFgO8YTiHMdTfz +Q2wf06TdXzOcM6XwPn3azyKmk7sn2v0s1pxGw8eu8tbmdU46xaLijwipn/B7NMsjk2gnqN +eptwb/SQmIZZcloQZYLx9PejarAHe2NJ5BSJqOrSHZHYXjiSKj4X55lGdDOVCUCf4lmStQ +qCS2LiM8Uhfga6f3X5EIBY75kqzmovDnPrqjufnCfYjBzQZ/m/txCbnZ9sTdQfXoVFAAAA +wQC/5nbU5HzZtg7bA3kfBRUNGUSl8nM2zENY9Rxc8sZiL7iH3s1HAVyz8Frvmc1Wgt6EF+ +WhtmNFklOmdYwq0W5+2qRdUN2P9QL+GKbuyp4AvwRmCFNhgm2GrCWQj/rkZ61vYS3bM8J8 +MNJglvU2FktXvwFODhf6Kv/7fZQnJCf2LTMG6hIKF4LdBOSS/0V5MH2v4xu2U64wqQAQnu +KzG4sRedsSHBGSknROJ7eGvGPZLh56PRb2gYPItoHcTMHqB6UAAADBAM4Xv6tHQFZtL+ul +FwVVKhr3EKGY3+RV9IBXvXDkhee4i594Yl67BFUSU4eDb4xuek24znwKn+UFERzp+1X02h +I1dZRdKtzJWOUQIF0FMPHbaPTuS7viT1OrL/PnG1yXUa+ii0qLExYI9qe463e8w6fNwhaT +Em2wiDZcxr8SjQfqimyfmDizVlLE/xdgJ56eJ+OyXLjpezKKQif9YcFUW/eHej3YEcok6o +WDwYpXG8z+VwOnOV4UN/sS8pLkJUdpqwAAAMEA7iTUVH3IvXMno0xYrVdhgNtZZGfQnryt +pRfG/f5eQ1tjEQwE31mrbBcR278YWlQZwrMWZ0hDaJ3Q/Cp0+JySlm17jsA33lnTRmCHF4 +WolX/LlFtH6jLr9SB8GOsn/8lC6IcvkED0UYYBjXipl6Unh9ZPnpbmJK9SgZWKNTGS1NBw +xcVWIZTKOrpCD+zWH6KCviuhd3J5vBgkSVxTUzDqA7TmnMnUUxDQAAaU7Eqtt3CJuxsJIs +vmZ2QrVK8TstC/AAAACm5jd0Bkb2dnZXI= +-----END OPENSSH PRIVATE KEY----- +#EOF +chmod 600 ${PRIVATE_KEY} + +cat >${PUBLIC_KEY} <<'#EOF' +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC/t7p34BvEQu9GTuxmKmR626qATj+9Vt//o8y453U5I/I46wA6zjW29zhKThL7/pF4dbiXUMrLkgfSyY6wDWTnyQoWQ3U0KEdTnPQKK0CJnB2XvOq/E0mqUyZiIZO5uFB0ycOurQACeHKv1MWtxmZg9yn20dym31VBZ4thH/Xq1PndaemBO4xaWDIznxukKH9wM7uA6NLJceE8/GtTWp2vmD/lP0cWBB+LYJfy+TikNj5DDUuenUwykpPiL3FYYxTygMkbfTxGvNSL23PVFN4+1uzjFc7Lxxc5loA4MNCiTiJa8kSkaAlguOE24fNmJQNx9zVKRe8426LuPlEL+uxA5gHlJ3sBOUXgfKYDFkneI+755gFmLuXAg966D7jZaVWif4Pl05iauBw+kBm/vrsu2G/rYUya9zVIccY9ktLd1Z9tl5drK9737AiQoUS4M3vrNgBq+NJGb18EqfLon6egmClsMxU2dN80Dbx+w1uwJLo6wGMtuIWMwfGG5gAIxpU= user@rclone-serve-test +#EOF +chmod 600 ${PUBLIC_KEY} + + run rclone serve sftp --authorized-keys "${PUBLIC_KEY}" --addr ${IP}:${PORT} ${DATADIR} + + echo type=sftp + echo ssh=ssh -i ${PRIVATE_KEY} -o StrictHostKeyChecking=no -p ${PORT} user@${IP} + echo _connect=${IP}:${PORT} +} + +. $(dirname "$0")/rclone-serve.bash diff --git a/fstest/testserver/init.d/TestSMB b/fstest/testserver/init.d/TestSMB new file mode 100755 index 0000000..4e10d59 --- /dev/null +++ b/fstest/testserver/init.d/TestSMB @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -e + +NAME=smb +USER=rclone +PASS=GNF3Cqeu +WORKGROUP=thepub +PORT=28630 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -p 127.0.0.1:${PORT}:445 \ + -p 127.0.0.1:${PORT}:445/udp \ + dperson/samba \ + -p \ + -u "rclone;${PASS}" \ + -w "${WORKGROUP}" \ + -s "public;/share" \ + -s "rclone;/rclone;yes;no;no;rclone" + + echo type=smb + echo host=127.0.0.1 + echo user=$USER + echo port=$PORT + echo pass=$(rclone obscure $PASS) + echo domain=$WORKGROUP + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSMBKerberos b/fstest/testserver/init.d/TestSMBKerberos new file mode 100755 index 0000000..f54eeb6 --- /dev/null +++ b/fstest/testserver/init.d/TestSMBKerberos @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -e + +IMAGE=rclone/test-smb-kerberos +NAME=smb-kerberos +USER=rclone +DOMAIN=RCLONE +REALM=RCLONE.LOCAL +SMB_PORT=28633 +KRB5_PORT=28634 + +# KRB5_CONFIG and KRB5CCNAME should be set by the caller but default +# them here for the integration tests +export TEMP_DIR=/tmp/rclone_krb5 +mkdir -p "${TEMP_DIR}" +export KRB5_CONFIG=${KRB5_CONFIG:-${TEMP_DIR}/krb5.conf} +export KRB5CCNAME=${KRB5CCNAME:-${TEMP_DIR}/ccache} + +. $(dirname "$0")/docker.bash + +start() { + docker build -t ${IMAGE} --load - <<EOF +FROM alpine:3.21 +RUN apk add --no-cache samba-dc +RUN rm -rf /etc/samba/smb.conf /var/lib/samba \ + && mkdir -p /var/lib/samba/private \ + && samba-tool domain provision \ + --use-rfc2307 \ + --option acl_xattr:security_acl_name=user.NTACL \ + --realm=$REALM \ + --domain=$DOMAIN \ + --server-role=dc \ + --dns-backend=SAMBA_INTERNAL \ + --host-name=localhost \ + && samba-tool user add --random-password $USER \ + && samba-tool user setexpiry $USER --noexpiry \ + && mkdir -m 777 /share /rclone \ + && cat <<EOS >> /etc/samba/smb.conf +[global] +server signing = auto +[public] +path = /share +browseable = yes +read only = yes +guest ok = yes +[rclone] +path = /rclone +browseable = yes +read only = no +guest ok = no +valid users = rclone +EOS +CMD ["samba", "-i"] +EOF + + docker run --rm -d --name ${NAME} \ + -p 127.0.0.1:${SMB_PORT}:445 \ + -p 127.0.0.1:${SMB_PORT}:445/udp \ + -p 127.0.0.1:${KRB5_PORT}:88 \ + ${IMAGE} + + # KRB5_CONFIG and KRB5CCNAME are set by the caller + cat > ${KRB5_CONFIG} <<EOF +[libdefaults] + default_realm = ${REALM} +[realms] +${REALM} = { + kdc = localhost +} +EOF + docker cp ${KRB5_CONFIG} ${NAME}:/etc/krb5.conf + docker exec ${NAME} samba-tool user get-kerberos-ticket rclone --output-krb5-ccache=/tmp/ccache + docker cp ${NAME}:/tmp/ccache ${KRB5CCNAME} + sed -i -e "s/localhost/localhost:${KRB5_PORT}/" ${KRB5_CONFIG} + + echo type=smb + echo host=localhost + echo port=$SMB_PORT + echo use_kerberos=true + echo _connect=127.0.0.1:${SMB_PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSMBKerberosCcache b/fstest/testserver/init.d/TestSMBKerberosCcache new file mode 100755 index 0000000..1238299 --- /dev/null +++ b/fstest/testserver/init.d/TestSMBKerberosCcache @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +set -e + +# Set default location for Kerberos config and ccache. Can be overridden by the caller +# using environment variables RCLONE_TEST_CUSTOM_CCACHE_LOCATION and KRB5_CONFIG. +export TEMP_DIR=/tmp/rclone_krb5_ccache +mkdir -p "${TEMP_DIR}" +export KRB5_CONFIG=${KRB5_CONFIG:-${TEMP_DIR}/krb5.conf} +export RCLONE_TEST_CUSTOM_CCACHE_LOCATION=${RCLONE_TEST_CUSTOM_CCACHE_LOCATION:-${TEMP_DIR}/ccache} + +IMAGE=rclone/test-smb-kerberos-ccache +NAME=smb-kerberos-ccache +USER=rclone +DOMAIN=RCLONE +REALM=RCLONE.LOCAL +SMB_PORT=28637 +KRB5_PORT=28638 + +. $(dirname "$0")/docker.bash + +start() { + docker build -t ${IMAGE} --load - <<EOF +FROM alpine:3.21 +RUN apk add --no-cache samba-dc +RUN rm -rf /etc/samba/smb.conf /var/lib/samba \ + && mkdir -p /var/lib/samba/private \ + && samba-tool domain provision \ + --use-rfc2307 \ + --option acl_xattr:security_acl_name=user.NTACL \ + --realm=$REALM \ + --domain=$DOMAIN \ + --server-role=dc \ + --dns-backend=SAMBA_INTERNAL \ + --host-name=localhost \ + && samba-tool user add --random-password $USER \ + && samba-tool user setexpiry $USER --noexpiry \ + && mkdir -m 777 /share /rclone \ + && cat <<EOS >> /etc/samba/smb.conf +[global] +server signing = auto +[public] +path = /share +browseable = yes +read only = yes +guest ok = yes +[rclone] +path = /rclone +browseable = yes +read only = no +guest ok = no +valid users = rclone +EOS +CMD ["samba", "-i"] +EOF + + docker run --rm -d --name ${NAME} \ + -p 127.0.0.1:${SMB_PORT}:445 \ + -p 127.0.0.1:${SMB_PORT}:445/udp \ + -p 127.0.0.1:${KRB5_PORT}:88 \ + ${IMAGE} + + cat > "${KRB5_CONFIG}" <<EOF +[libdefaults] + default_realm = ${REALM} +[realms] +${REALM} = { + kdc = localhost +} +EOF + + docker cp "${KRB5_CONFIG}" ${NAME}:/etc/krb5.conf + docker exec ${NAME} samba-tool user get-kerberos-ticket rclone --output-krb5-ccache=/tmp/ccache + docker cp ${NAME}:/tmp/ccache "${RCLONE_TEST_CUSTOM_CCACHE_LOCATION}" + sed -i -e "s/localhost/localhost:${KRB5_PORT}/" "${KRB5_CONFIG}" + + echo type=smb + echo host=localhost + echo port=$SMB_PORT + echo use_kerberos=true + echo kerberos_ccache=${RCLONE_TEST_CUSTOM_CCACHE_LOCATION} + echo _connect=127.0.0.1:${SMB_PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSeafile b/fstest/testserver/init.d/TestSeafile new file mode 100755 index 0000000..553e9d0 --- /dev/null +++ b/fstest/testserver/init.d/TestSeafile @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -e + +# environment variables passed on docker-compose +export NAME=seafile7 +export MYSQL_ROOT_PASSWORD=pixenij4zacoguq0kopamid6 +export SEAFILE_ADMIN_EMAIL=seafile@rclone.org +export SEAFILE_ADMIN_PASSWORD=pixenij4zacoguq0kopamid6 +export SEAFILE_IP=127.0.0.1 +export SEAFILE_PORT=8087 +export SEAFILE_TEST_DATA=${SEAFILE_TEST_DATA:-/tmp/seafile-test-data} +export SEAFILE_VERSION=latest + +# make sure the data directory exists +mkdir -p ${SEAFILE_TEST_DATA}/${NAME} + +# docker-compose project directory +COMPOSE_DIR=$(dirname "$0")/seafile + +start() { + docker-compose --project-directory ${COMPOSE_DIR} --project-name ${NAME} --file ${COMPOSE_DIR}/docker-compose.yml up -d + + # wait for Seafile server to start + seafile_endpoint="http://${SEAFILE_IP}:${SEAFILE_PORT}/" + wait_seconds=1 + echo -n "Waiting for Seafile server to start" + for iterations in `seq 1 60`; + do + http_code=$(curl -s -o /dev/null -L -w '%{http_code}' "$seafile_endpoint" || true;) + if [ "$http_code" -eq 200 ]; then + echo + break + fi + echo -n "." + sleep $wait_seconds + done + + # authentication token answer should be like: {"token":"dbf58423f1632b5b679a13b0929f1d0751d9250c"} + TOKEN=`curl --silent \ + --data-urlencode username=${SEAFILE_ADMIN_EMAIL} -d password=${SEAFILE_ADMIN_PASSWORD} \ + http://${SEAFILE_IP}:${SEAFILE_PORT}/api2/auth-token/ \ + | sed 's/^{"token":"\(.*\)"}$/\1/'` + + # create default library + curl --silent -o /dev/null -X POST -H "Authorization: Token ${TOKEN}" "http://${SEAFILE_IP}:${SEAFILE_PORT}/api2/default-repo/" + + echo _connect=${SEAFILE_IP}:${SEAFILE_PORT} + echo type=seafile + echo url=http://${SEAFILE_IP}:${SEAFILE_PORT}/ + echo user=${SEAFILE_ADMIN_EMAIL} + echo pass=$(rclone obscure ${SEAFILE_ADMIN_PASSWORD}) + echo library=My Library +} + +stop() { + if status ; then + docker-compose --project-directory ${COMPOSE_DIR} --project-name ${NAME} --file ${COMPOSE_DIR}/docker-compose.yml down + fi +} + +status() { + if docker ps --format "{{.Names}}" | grep ^${NAME}_seafile_1$ >/dev/null ; then + echo "$NAME running" + else + echo "$NAME not running" + return 1 + fi + return 0 +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSeafileEncrypted b/fstest/testserver/init.d/TestSeafileEncrypted new file mode 100755 index 0000000..dd4b6bc --- /dev/null +++ b/fstest/testserver/init.d/TestSeafileEncrypted @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -e + +# local variables +TEST_LIBRARY=Encrypted +TEST_LIBRARY_PASSWORD=SecretKey + +# environment variables passed on docker-compose +export NAME=seafile7encrypted +export MYSQL_ROOT_PASSWORD=pixenij4zacoguq0kopamid6 +export SEAFILE_ADMIN_EMAIL=seafile@rclone.org +export SEAFILE_ADMIN_PASSWORD=pixenij4zacoguq0kopamid6 +export SEAFILE_IP=127.0.0.1 +export SEAFILE_PORT=8088 +export SEAFILE_TEST_DATA=${SEAFILE_TEST_DATA:-/tmp/seafile-test-data} +export SEAFILE_VERSION=latest + +# make sure the data directory exists +mkdir -p ${SEAFILE_TEST_DATA}/${NAME} + +# docker-compose project directory +COMPOSE_DIR=$(dirname "$0")/seafile + +start() { + docker-compose --project-directory ${COMPOSE_DIR} --project-name ${NAME} --file ${COMPOSE_DIR}/docker-compose.yml up -d + + # it takes some time for the database to be created + sleep 60 + + # authentication token answer should be like: {"token":"dbf58423f1632b5b679a13b0929f1d0751d9250c"} + TOKEN=`curl --silent \ + --data-urlencode username=${SEAFILE_ADMIN_EMAIL} -d password=${SEAFILE_ADMIN_PASSWORD} \ + http://${SEAFILE_IP}:${SEAFILE_PORT}/api2/auth-token/ \ + | sed 's/^{"token":"\(.*\)"}$/\1/'` + + # create encrypted library + curl --silent -o /dev/null -X POST -d "name=${TEST_LIBRARY}&passwd=${TEST_LIBRARY_PASSWORD}" -H "Authorization: Token ${TOKEN}" "http://${SEAFILE_IP}:${SEAFILE_PORT}/api2/repos/" + + echo _connect=${SEAFILE_IP}:${SEAFILE_PORT} + echo type=seafile + echo url=http://${SEAFILE_IP}:${SEAFILE_PORT}/ + echo user=${SEAFILE_ADMIN_EMAIL} + echo pass=$(rclone obscure ${SEAFILE_ADMIN_PASSWORD}) + echo library=${TEST_LIBRARY} + echo library_key=$(rclone obscure ${TEST_LIBRARY_PASSWORD}) +} + +stop() { + if status ; then + docker-compose --project-directory ${COMPOSE_DIR} --project-name ${NAME} --file ${COMPOSE_DIR}/docker-compose.yml down + fi +} + +status() { + if docker ps --format "{{.Names}}" | grep ^${NAME}_seafile_1$ >/dev/null ; then + echo "$NAME running" + else + echo "$NAME not running" + return 1 + fi + return 0 +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSeafileV6 b/fstest/testserver/init.d/TestSeafileV6 new file mode 100755 index 0000000..e2365aa --- /dev/null +++ b/fstest/testserver/init.d/TestSeafileV6 @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +set -e + +# local variables +NAME=seafile6 +SEAFILE_IP=127.0.0.1 +SEAFILE_PORT=8086 +SEAFILE_ADMIN_EMAIL=seafile@rclone.org +SEAFILE_ADMIN_PASSWORD=qebiwob7wafixif8sojiboj4 +SEAFILE_TEST_DATA=${SEAFILE_TEST_DATA:-/tmp/seafile-test-data} +SEAFILE_VERSION=latest + +. $(dirname "$0")/docker.bash + +start() { + # make sure the data directory exists + mkdir -p ${SEAFILE_TEST_DATA}/${NAME} + + docker run --rm -d --name $NAME \ + -e SEAFILE_SERVER_HOSTNAME=${SEAFILE_IP}:${SEAFILE_PORT} \ + -e SEAFILE_ADMIN_EMAIL=${SEAFILE_ADMIN_EMAIL} \ + -e SEAFILE_ADMIN_PASSWORD=${SEAFILE_ADMIN_PASSWORD} \ + -v ${SEAFILE_TEST_DATA}/${NAME}:/shared \ + -p ${SEAFILE_IP}:${SEAFILE_PORT}:80 \ + seafileltd/seafile:${SEAFILE_VERSION} + + # it takes some time for the database to be created + sleep 60 + + # authentication token answer should be like: {"token":"dbf58423f1632b5b679a13b0929f1d0751d9250c"} + TOKEN=`curl --silent \ + --data-urlencode username=${SEAFILE_ADMIN_EMAIL} -d password=${SEAFILE_ADMIN_PASSWORD} \ + http://${SEAFILE_IP}:${SEAFILE_PORT}/api2/auth-token/ \ + | sed 's/^{"token":"\(.*\)"}$/\1/'` + + # create default library + curl --silent -o /dev/null -X POST -H "Authorization: Token ${TOKEN}" "http://${SEAFILE_IP}:${SEAFILE_PORT}/api2/default-repo/" + + echo _connect=${SEAFILE_IP}:${SEAFILE_PORT} + echo type=seafile + echo url=http://${SEAFILE_IP}:${SEAFILE_PORT}/ + echo user=${SEAFILE_ADMIN_EMAIL} + echo pass=$(rclone obscure ${SEAFILE_ADMIN_PASSWORD}) + echo library=My Library +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSia b/fstest/testserver/init.d/TestSia new file mode 100755 index 0000000..9b11caf --- /dev/null +++ b/fstest/testserver/init.d/TestSia @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -e + +NAME=Sia + +# shellcheck disable=SC1090 +. "$(dirname "$0")"/docker.bash + +# wait until Sia test network is up, +# the Sia renter forms contracts on the blockchain +# and the renter is upload ready +wait_for_sia() { + until curl -A Sia-Agent -s "$1" | grep -q '"ready":true' + do + sleep 5 + done +} +export -f wait_for_sia + +start() { + # use non-production sia port in test + SIA_CONN="127.0.0.1:39980" + # nebulouslabs/siaantfarm is stale, use up-to-date image + ANTFARM_IMAGE=ivandeex/sia-antfarm:latest + + # pull latest antfarm image (dont use local image) + docker pull --quiet $ANTFARM_IMAGE + + # start latest antfarm with default config + docker run --rm --detach --name "$NAME" \ + --publish "${SIA_CONN}:9980" \ + $ANTFARM_IMAGE + + # wait until the test network is upload ready + timeout 300 bash -c "wait_for_sia ${SIA_CONN}/renter/uploadready" + + # confirm backend type in the generated rclone.conf + echo "type=sia" + # override keys in the Sia section of generated rclone.conf + echo "api_url=http://${SIA_CONN}/" + # hint test harness where to probe for connection + echo "_connect=${SIA_CONN}" +} + +stop() { + if status ; then + docker logs "$NAME" >> sia-test.log 2>&1 + docker kill "$NAME" + echo "${NAME} stopped" + fi +} + +# shellcheck disable=SC1090 +. "$(dirname "$0")"/run.bash diff --git a/fstest/testserver/init.d/TestSwiftAIO b/fstest/testserver/init.d/TestSwiftAIO new file mode 100755 index 0000000..7e20bd6 --- /dev/null +++ b/fstest/testserver/init.d/TestSwiftAIO @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -e + +NAME=swift-aio +PORT=28628 + +. $(dirname "$0")/docker.bash + +start() { + # We need to replace the remakerings in the container to create Policy-1. + docker run --rm -d --name ${NAME} \ + -p 127.0.0.1:${PORT}:8080 \ + -v $(dirname "$0")/TestSwiftAIO.d/remakerings:/etc/swift/remakerings:ro \ + openstackswift/saio + + echo type=swift + echo env_auth=false + echo user=test:tester + echo key=testing + echo auth=http://127.0.0.1:${PORT}/auth/v1.0 + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestSwiftAIO.d/remakerings b/fstest/testserver/init.d/TestSwiftAIO.d/remakerings new file mode 100755 index 0000000..27c49b1 --- /dev/null +++ b/fstest/testserver/init.d/TestSwiftAIO.d/remakerings @@ -0,0 +1,46 @@ +#!/bin/sh + +if ! grep -q "^\[storage-policy:1\]" swift.conf; then + cat <<EOF >> swift.conf + +[storage-policy:1] +name = Policy-1 +EOF +fi + +rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz + +swift-ring-builder object.builder create 10 1 1 +swift-ring-builder object.builder add r1z1-127.0.0.1:6200/swift-d0 1 +swift-ring-builder object.builder add r1z1-127.0.0.1:6200/swift-d1 1 +swift-ring-builder object.builder add r1z1-127.0.0.1:6200/swift-d2 1 +swift-ring-builder object.builder add r1z1-127.0.0.1:6200/swift-d3 1 +swift-ring-builder object.builder add r1z1-127.0.0.1:6200/swift-d4 1 +swift-ring-builder object.builder add r1z1-127.0.0.1:6200/swift-d5 1 +swift-ring-builder object.builder rebalance +swift-ring-builder container.builder create 10 1 1 +swift-ring-builder container.builder add r1z1-127.0.0.1:6201/swift-d0 1 +swift-ring-builder container.builder add r1z1-127.0.0.1:6201/swift-d1 1 +swift-ring-builder container.builder add r1z1-127.0.0.1:6201/swift-d2 1 +swift-ring-builder container.builder add r1z1-127.0.0.1:6201/swift-d3 1 +swift-ring-builder container.builder add r1z1-127.0.0.1:6201/swift-d4 1 +swift-ring-builder container.builder add r1z1-127.0.0.1:6201/swift-d5 1 +swift-ring-builder container.builder rebalance +swift-ring-builder account.builder create 10 1 1 +swift-ring-builder account.builder add r1z1-127.0.0.1:6202/swift-d0 1 +swift-ring-builder account.builder add r1z1-127.0.0.1:6202/swift-d1 1 +swift-ring-builder account.builder add r1z1-127.0.0.1:6202/swift-d2 1 +swift-ring-builder account.builder add r1z1-127.0.0.1:6202/swift-d3 1 +swift-ring-builder account.builder add r1z1-127.0.0.1:6202/swift-d4 1 +swift-ring-builder account.builder add r1z1-127.0.0.1:6202/swift-d5 1 +swift-ring-builder account.builder rebalance + +# For Policy-1: +swift-ring-builder object-1.builder create 10 1 1 +swift-ring-builder object-1.builder add r1z1-127.0.0.1:6200/swift-d0 1 +swift-ring-builder object-1.builder add r1z1-127.0.0.1:6200/swift-d1 1 +swift-ring-builder object-1.builder add r1z1-127.0.0.1:6200/swift-d2 1 +swift-ring-builder object-1.builder add r1z1-127.0.0.1:6200/swift-d3 1 +swift-ring-builder object-1.builder add r1z1-127.0.0.1:6200/swift-d4 1 +swift-ring-builder object-1.builder add r1z1-127.0.0.1:6200/swift-d5 1 +swift-ring-builder object-1.builder rebalance diff --git a/fstest/testserver/init.d/TestSwiftAIOsegments b/fstest/testserver/init.d/TestSwiftAIOsegments new file mode 100755 index 0000000..db02630 --- /dev/null +++ b/fstest/testserver/init.d/TestSwiftAIOsegments @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +NAME=swift-aio-segments +PORT=28632 + +. $(dirname "$0")/docker.bash + +start() { + # We need to replace the remakerings in the container to create Policy-1. + docker run --rm -d --name ${NAME} \ + -p 127.0.0.1:${PORT}:8080 \ + -v $(dirname "$0")/TestSwiftAIO.d/remakerings:/etc/swift/remakerings:ro \ + openstackswift/saio + + echo type=swift + echo env_auth=false + echo user=test:tester + echo key=testing + echo auth=http://127.0.0.1:${PORT}/auth/v1.0 + echo use_segments_container=false + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestWebdavInfiniteScale b/fstest/testserver/init.d/TestWebdavInfiniteScale new file mode 100755 index 0000000..f13e22b --- /dev/null +++ b/fstest/testserver/init.d/TestWebdavInfiniteScale @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -e + +NAME=infinitescale +USER=admin +PASS=admin +PORT=28639 +CONF_DIR=/tmp/ocis-config +mkdir -p ${CONF_DIR} +chmod 777 ${CONF_DIR} || true + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm --name $NAME \ + -v ${CONF_DIR}:/etc/ocis \ + -e "OCIS_INSECURE=true" \ + -e "IDM_ADMIN_PASSWORD=$PASS" \ + -e "OCIS_FORCE_CONFIG_OVERWRITE=true" \ + -e "OCIS_URL=https://127.0.0.1:$PORT" \ + owncloud/ocis \ + init + + docker run --rm -d --name $NAME \ + -e "OCIS_LOG_LEVEL=debug" \ + -e "OCIS_LOG_PRETTY=true" \ + -e "OCIS_URL=https://127.0.0.1:$PORT" \ + -e "OCIS_ADMIN_USER_ID=some-admin-user-id-0000-100000000000" \ + -e "IDM_ADMIN_PASSWORD=$PASS" \ + -e "OCIS_INSECURE=true" \ + -e "PROXY_ENABLE_BASIC_AUTH=true" \ + -v ${CONF_DIR}:/etc/ocis \ + -p 127.0.0.1:${PORT}:9200 \ + owncloud/ocis + + echo type=webdav + echo url=https://127.0.0.1:${PORT}/dav/spaces/some-admin-user-id-0000-100000000000 + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo vendor=infinitescale + echo _connect=127.0.0.1:${PORT} + echo _connect_delay=5s +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestWebdavNextcloud b/fstest/testserver/init.d/TestWebdavNextcloud new file mode 100755 index 0000000..42766e5 --- /dev/null +++ b/fstest/testserver/init.d/TestWebdavNextcloud @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -e + +NAME=nextcloud +USER=rclone +PASS=ArmorAbleMale6 +PORT=28629 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "SQLITE_DATABASE=nextcloud.db" \ + -e "NEXTCLOUD_ADMIN_USER=rclone" \ + -e "NEXTCLOUD_ADMIN_PASSWORD=$PASS" \ + -e "NEXTCLOUD_TRUSTED_DOMAINS=*.*.*.*" \ + -p 127.0.0.1:${PORT}:80 \ + nextcloud:latest + + echo type=webdav + echo url=http://127.0.0.1:${PORT}/remote.php/dav/files/$USER/ + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo vendor=nextcloud + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestWebdavOwncloud b/fstest/testserver/init.d/TestWebdavOwncloud new file mode 100755 index 0000000..d2dc238 --- /dev/null +++ b/fstest/testserver/init.d/TestWebdavOwncloud @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -e + +NAME=owncloud +USER=rclone +PASS=HarperGrayerFewest5 +PORT=38081 + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME \ + -e "OWNCLOUD_DOMAIN=localhost:8080" \ + -e "OWNCLOUD_DB_TYPE=sqlite" \ + -e "OWNCLOUD_DB_NAME=owncloud.db" \ + -e "OWNCLOUD_ADMIN_USERNAME=$USER" \ + -e "OWNCLOUD_ADMIN_PASSWORD=$PASS" \ + -e "OWNCLOUD_MYSQL_UTF8MB4=true" \ + -e "OWNCLOUD_REDIS_ENABLED=false" \ + -e "OWNCLOUD_TRUSTED_DOMAINS=127.0.0.1" \ + -p 127.0.0.1:${PORT}:8080 \ + owncloud/server + + echo type=webdav + echo url=http://127.0.0.1:${PORT}/remote.php/webdav/ + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo vendor=owncloud + echo _connect=127.0.0.1:${PORT} +} + +. $(dirname "$0")/run.bash diff --git a/fstest/testserver/init.d/TestWebdavRclone b/fstest/testserver/init.d/TestWebdavRclone new file mode 100755 index 0000000..e740ecc --- /dev/null +++ b/fstest/testserver/init.d/TestWebdavRclone @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +NAME=rclone-serve-webdav +USER=rclone +PASS=PagansSwimExpiry9 +IP=127.0.0.1 +PORT=28620 + +start() { + run rclone serve webdav --user $USER --pass $PASS --addr ${IP}:${PORT} ${DATADIR} + + echo type=webdav + echo vendor=rclone + echo url=http://${IP}:${PORT}/ + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo _connect=${IP}:$PORT +} + +. $(dirname "$0")/rclone-serve.bash diff --git a/fstest/testserver/init.d/docker.bash b/fstest/testserver/init.d/docker.bash new file mode 100644 index 0000000..1bcc2c2 --- /dev/null +++ b/fstest/testserver/init.d/docker.bash @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +stop() { + if status ; then + docker stop "$NAME" + echo "$NAME stopped" + fi +} + +status() { + if docker ps --format '{{.Names}}' | grep -q "^${NAME}$" ; then + echo "$NAME running" + else + echo "$NAME not running" + return 1 + fi + return 0 +} + +docker_ip() { + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{"\n"}}{{end}}' "$NAME" | head -n 1 +} diff --git a/fstest/testserver/init.d/rclone-serve.bash b/fstest/testserver/init.d/rclone-serve.bash new file mode 100644 index 0000000..408960e --- /dev/null +++ b/fstest/testserver/init.d/rclone-serve.bash @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# start an "rclone serve" server + +PIDFILE=/tmp/${NAME}.pid +DATADIR=/tmp/${NAME}-data + +stop() { + if status ; then + pid=$(cat "$PIDFILE") + kill "$pid" + rm "$PIDFILE" + echo "$NAME stopped" + fi +} + +status() { + if [ -e "$PIDFILE" ]; then + pid=$(cat "$PIDFILE") + if kill -0 "$pid" >/dev/null 2>&1; then + # echo "$NAME running" + return 0 + else + rm "$PIDFILE" + fi + fi + # echo "$NAME not running" + return 1 +} + +run() { + if ! status ; then + mkdir -p "$DATADIR" + nohup "$@" >> "/tmp/${NAME}.log" 2>&1 </dev/null & + pid=$! + echo $pid > "$PIDFILE" + disown "$pid" + fi +} + +# shellcheck disable=SC1090 +. "$(dirname "$0")/run.bash" diff --git a/fstest/testserver/init.d/run.bash b/fstest/testserver/init.d/run.bash new file mode 100644 index 0000000..ff1f70e --- /dev/null +++ b/fstest/testserver/init.d/run.bash @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail + +RUN_BASE="${STATE_DIR:-${XDG_RUNTIME_DIR:-/tmp}/rclone-test-server}" +: "${NAME:=$(basename "$0")}" +RUN_ROOT="${RUN_BASE}/${NAME}" +RUN_STATE="${RUN_ROOT}/state" +RUN_LOCK_FILE="${RUN_ROOT}/lock" +RUN_REF_COUNT="${RUN_STATE}/refcount" +RUN_OUTPUT="${RUN_STATE}/env" + +mkdir -p "${RUN_STATE}" +[[ -f "${RUN_REF_COUNT}" ]] || echo 0 >"${RUN_REF_COUNT}" +[[ -f "${RUN_OUTPUT}" ]] || : >"${RUN_OUTPUT}" +: > "${RUN_LOCK_FILE}" # ensure file exists + +# status helper that won't trip set -e +_is_running() { set +e; status >/dev/null 2>&1; local rc=$?; set -e; return $rc; } + +_acquire_lock() { + # open fd 9 on lock file and take exclusive lock + exec 9>"${RUN_LOCK_FILE}" + flock -x 9 +} + +_release_lock() { + flock -u 9 + exec 9>&- +} + +case "${1:-}" in + start) + _acquire_lock + trap '_release_lock' EXIT + + rc=$(cat "${RUN_REF_COUNT}" 2>/dev/null || echo 0) + + if (( rc == 0 )); then + # First client: ensure a clean instance, then start and cache env + if _is_running; then + stop || true + fi + if ! out="$(start)"; then + echo "failed to start" >&2 + exit 1 + fi + printf "%s\n" "$out" > "${RUN_OUTPUT}" + else + # Already owned: make sure it’s still up; if not, restart and refresh env + if ! _is_running; then + if ! out="$(start)"; then + echo "failed to restart" >&2 + exit 1 + fi + printf "%s\n" "$out" > "${RUN_OUTPUT}" + fi + fi + + rc=$((rc+1)); echo "${rc}" > "${RUN_REF_COUNT}" + cat "${RUN_OUTPUT}" + + trap - EXIT + _release_lock + ;; + + stop) + _acquire_lock + trap '_release_lock' EXIT + + rc=$(cat "${RUN_REF_COUNT}" 2>/dev/null || echo 0) + if (( rc > 0 )); then rc=$((rc-1)); fi + echo "${rc}" > "${RUN_REF_COUNT}" + if (( rc == 0 )) && _is_running; then + stop || true + fi + + trap - EXIT + _release_lock + ;; + + reset) + _acquire_lock + trap '_release_lock' EXIT + + stop || true + rm -rf "${RUN_BASE}" + + trap - EXIT + _release_lock + ;; + + status) + # passthrough; do NOT take the lock + status + ;; + + *) + echo "usage: $0 {start|stop|reset|status}" >&2 + exit 2 + ;; +esac diff --git a/fstest/testserver/init.d/seafile/docker-compose.yml b/fstest/testserver/init.d/seafile/docker-compose.yml new file mode 100644 index 0000000..8e0a099 --- /dev/null +++ b/fstest/testserver/init.d/seafile/docker-compose.yml @@ -0,0 +1,31 @@ +version: '2.0' +services: + db: + image: mariadb:10.5 + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_LOG_CONSOLE=true + volumes: + - ${SEAFILE_TEST_DATA}/${NAME}/seafile-mysql/db:/var/lib/mysql + + memcached: + image: memcached:1.6.9 + entrypoint: memcached -m 256 + + seafile: + image: seafileltd/seafile-mc:${SEAFILE_VERSION} + ports: + - "${SEAFILE_IP}:${SEAFILE_PORT}:80" + volumes: + - ${SEAFILE_TEST_DATA}/${NAME}/seafile-data:/shared + environment: + - DB_HOST=db + - DB_ROOT_PASSWD=${MYSQL_ROOT_PASSWORD} + - TIME_ZONE=Etc/UTC + - SEAFILE_ADMIN_EMAIL=${SEAFILE_ADMIN_EMAIL} + - SEAFILE_ADMIN_PASSWORD=${SEAFILE_ADMIN_PASSWORD} + - SEAFILE_SERVER_LETSENCRYPT=false + - SEAFILE_SERVER_HOSTNAME=${SEAFILE_IP}:${SEAFILE_PORT} + depends_on: + - db + - memcached diff --git a/fstest/testserver/testserver.go b/fstest/testserver/testserver.go new file mode 100644 index 0000000..9967bf5 --- /dev/null +++ b/fstest/testserver/testserver.go @@ -0,0 +1,198 @@ +// Package testserver starts and stops test servers if required +package testserver + +import ( + "bytes" + "errors" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/fspath" +) + +var ( + findConfigOnce sync.Once + configDir string // where the config is stored +) + +// Assume we are run somewhere within the rclone root +func findConfig() (string, error) { + dir := filepath.Join("fstest", "testserver", "init.d") + for range 5 { + fi, err := os.Stat(dir) + if err == nil && fi.IsDir() { + return filepath.Abs(dir) + } else if !os.IsNotExist(err) { + return "", err + } + dir = filepath.Join("..", dir) + } + return "", errors.New("couldn't find testserver config files - run from within rclone source") +} + +// returns path to a script to start this server +func cmdPath(name string) string { + return filepath.Join(configDir, name) +} + +// return true if the server with name has a start command +func hasStartCommand(name string) bool { + fi, err := os.Stat(cmdPath(name)) + return err == nil && !fi.IsDir() +} + +// run the command returning the output and an error +func run(name, command string) (out []byte, err error) { + script := cmdPath(name) + cmd := exec.Command(script, command) + out, err = cmd.CombinedOutput() + if err != nil { + err = fmt.Errorf("failed to run %s %s\n%s: %w", script, command, string(out), err) + } + return out, err +} + +// envKey returns the environment variable name to set name, key +func envKey(name, key string) string { + return fmt.Sprintf("RCLONE_CONFIG_%s_%s", strings.ToUpper(name), strings.ToUpper(key)) +} + +// match a line of config var=value +var matchLine = regexp.MustCompile(`^([a-zA-Z_]+)=(.*)$`) + +// Start the server and env vars so rclone can use it +func start(name string) error { + fs.Logf(name, "Starting server") + out, err := run(name, "start") + if err != nil { + return err + } + // parse the output and set environment vars from it + var connect string + var connectDelay time.Duration + for line := range bytes.SplitSeq(out, []byte("\n")) { + line = bytes.TrimSpace(line) + part := matchLine.FindSubmatch(line) + if part != nil { + key, value := part[1], part[2] + if string(key) == "_connect" { + connect = string(value) + continue + } else if string(key) == "_connect_delay" { + connectDelay, err = time.ParseDuration(string(value)) + if err != nil { + return fmt.Errorf("bad _connect_delay: %w", err) + } + continue + } + + // fs.Debugf(name, "key = %q, envKey = %q, value = %q", key, envKey(name, string(key)), value) + err = os.Setenv(envKey(name, string(key)), string(value)) + if err != nil { + return err + } + } + } + if connect == "" { + fs.Logf(name, "Started server") + return nil + } + // If we got a _connect value then try to connect to it + const maxTries = 100 + var rdBuf = make([]byte, 1) + for i := 1; i <= maxTries; i++ { + if i != 0 { + time.Sleep(time.Second) + } + fs.Logf(name, "Attempting to connect to %q try %d/%d", connect, i, maxTries) + conn, err := net.DialTimeout("tcp", connect, time.Second) + if err != nil { + fs.Debugf(name, "Connection to %q failed try %d/%d: %v", connect, i, maxTries, err) + continue + } + + err = conn.SetReadDeadline(time.Now().Add(time.Second)) + if err != nil { + return fmt.Errorf("failed to set deadline: %w", err) + } + n, err := conn.Read(rdBuf) + _ = conn.Close() + fs.Debugf(name, "Read %d, error: %v", n, err) + if err != nil && !errors.Is(err, os.ErrDeadlineExceeded) { + // Try again + continue + } + if connectDelay > 0 { + fs.Logf(name, "Connect delay %v", connectDelay) + time.Sleep(connectDelay) + } + fs.Logf(name, "Started server and connected to %q", connect) + return nil + } + return fmt.Errorf("failed to connect to %q on %q", name, connect) +} + +// Stops the named test server +func stop(name string) { + fs.Logf(name, "Stopping server") + _, err := run(name, "stop") + if err != nil { + fs.Errorf(name, "Failed to stop server: %v", err) + } +} + +// No server to stop so do nothing +func stopNothing() { +} + +// Start starts the test server for remoteName. +// +// This must be stopped by calling the function returned when finished. +func Start(remote string) (fn func(), err error) { + // don't start the local backend + if remote == "" { + return stopNothing, nil + } + parsed, err := fspath.Parse(remote) + if err != nil { + return nil, err + } + name := parsed.ConfigString + // don't start the local backend + if name == "" { + return stopNothing, nil + } + + // Make sure we know where the config is + findConfigOnce.Do(func() { + configDir, err = findConfig() + }) + if err != nil { + return nil, err + } + + // If remote has no start command then do nothing + if !hasStartCommand(name) { + return stopNothing, nil + } + + // Start the server + err = start(name) + if err != nil { + return nil, err + } + + // And return a function to stop it + return func() { + stop(name) + }, nil + +} |