Add integration tests (#204)

Work in progress, some of the code is cribbed from
https://github.com/radian-software/dumbparens

7 files changed, 471 insertions, 141 deletions

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 0c8e88a..6e3948f 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -1,4 +1,4 @@
-name: Lint
+name: Tests and linters
 on:
   push:
     branches:
@@ -22,4 +22,4 @@ jobs:
         env:
           VERSION: ${{ matrix.emacs_version }}
         run: >-
-          make docker CMD="make unit lint lint-changelog"
+          make docker CMD="make unit integration lint lint-changelog"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0ca6cc3..f03780b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,7 +4,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog].
 
 ## Unreleased
+### Changes
+* Custom Emacs Lisp formatting functions have the option to report an
+  error asynchronously by invoking their callback with an error as
+  argument. Passing nil as argument indicates that there was no error,
+  as before. The old calling convention is still supported for
+  backwards compatibility, and errors can also be reported by
+  throwing, as normal. Implemented in [#204].
+
 ### Enhancements
+* There is a new keyword argument to `apheleia-format-buffer` which is
+  a more powerful callback that is guaranteed to be called except in
+  cases of synchronous nonlocal exit. See the docstring for details.
+  The old callback, which is only invoked on success and receives no
+  information about errors, is still supported and will continue to be
+  called if provided. See [#204].
+
 ### Formatters
 ### Bugs fixed
 * The point alignment algorithm, which has been slightly wrong since
@@ -16,6 +31,14 @@ The format is based on [Keep a Changelog].
 * [Formatter scripts](scripts/formatters) will now work on Windows if Emacs
   can find the executable defined in the shebang.
 
+### Internal
+* Major internal refactoring has occurred to make it possible to write
+  integration tests against Apheleia. This should improve future
+  stability but could have introduced some bugs in the initial
+  version. See [#204].
+* Some debugging log messages have changed, see [#204].
+
+[#204]: https://github.com/radian-software/apheleia/pull/204
 [#286]: https://github.com/radian-software/apheleia/pull/286
 [#285]: https://github.com/radian-software/apheleia/issues/285
 [#290]: https://github.com/radian-software/apheleia/pull/290
diff --git a/Makefile b/Makefile
index ce9e944..0d54fcd 100644
--- a/Makefile
+++ b/Makefile
@@ -127,3 +127,10 @@ $(BUTTERCUP):
 .PHONY: unit
 unit: $(BUTTERCUP) ## Run unit tests
 	@$(BUTTERCUP)/bin/buttercup test/unit -L $(BUTTERCUP) -L .
+
+APHELEIA_IT := -L test/integration \
+	--eval "(setq apheleia-log-debug-info t)" -l apheleia-it
+
+.PHONY: integration
+integration: ## Run integration tests
+	@test/shared/run-func.bash apheleia-it-run-all-tests $(APHELEIA_IT)
diff --git a/apheleia-formatters.el b/apheleia-formatters.el
index 0a35876..a96272b 100644
--- a/apheleia-formatters.el
+++ b/apheleia-formatters.el
@@ -608,14 +608,16 @@ NO-QUERY, and CONNECTION-TYPE."
 (cl-defun apheleia--execute-formatter-process
     (&key ctx callback ensure exit-status)
   "Wrapper for `make-process' that behaves a bit more nicely.
-CTX is a formatter process context (see `apheleia-formatter--context').
-CALLBACK is invoked with one argument, the buffer containing the text
-from stdout, when the process terminates (if it succeeds). ENSURE is a
-callback that's invoked whether the process exited successfully or
-not. EXIT-STATUS is a function which is called with the exit
-status of the command; it should return non-nil to indicate that
-the command succeeded. If EXIT-STATUS is omitted, then the
-command succeeds provided that its exit status is 0."
+CTX is a formatter process context (see
+`apheleia-formatter--context'). CALLBACK is invoked with two
+arguments. The first is an error or nil. The second is the buffer
+containing the text from stdout, when the process terminates (if
+it succeeds). ENSURE is a callback that's invoked whether the
+process exited successfully or not. EXIT-STATUS is a function
+which is called with the exit status of the command; it should
+return non-nil to indicate that the command succeeded. If
+EXIT-STATUS is omitted, then the command succeeds provided that
+its exit status is 0."
   (apheleia--log
    'process "Trying to execute formatter process %s with %S"
    (apheleia-formatter--name ctx)
@@ -678,7 +680,7 @@ command succeeds provided that its exit status is 0."
                        (apheleia-log--formatter-result
                         ctx
                         log-name
-                        (apheleia-formatter--exit-status ctx)
+                        exit-ok
                         (buffer-local-value 'default-directory stdout)
                         (with-current-buffer stderr
                           (string-trim (buffer-string)))))
@@ -693,22 +695,29 @@ command succeeds provided that its exit status is 0."
                         :log (get-buffer log-name)))
                      (unwind-protect
                          (if exit-ok
-                             (when callback
-                               (apheleia--log
-                                'process
-                                (concat "Invoking process callback due "
-                                        "to successful exit status"))
-                               (funcall callback stdout))
-                           (message
-                            (concat
-                             "Failed to run %s: exit status %s "
-                             "(see %s %s)")
-                            (apheleia-formatter--arg1 ctx)
-                            proc-exit-status
-                            (if (string-prefix-p " " log-name)
-                                "hidden buffer"
-                              "buffer")
-                            (string-trim log-name)))
+                             (funcall callback nil stdout)
+                           (let ((errmsg
+                                  (format
+                                   (concat
+                                    "Failed to run %s: exit status %s "
+                                    "(see %s %s)")
+                                   (apheleia-formatter--arg1 ctx)
+                                   proc-exit-status
+                                   (if (string-prefix-p " " log-name)
+                                       "hidden buffer"
+                                     "buffer")
+                                   (string-trim log-name))))
+                             (message "%s" errmsg)
+                             (when noninteractive
+                               (message
+                                "%s"
+                                (concat
+                                 "(log buffer shown"
+                                 " below in batch mode)\n"
+                                 (with-current-buffer log-name
+                                   (buffer-string)))))
+                             (funcall
+                              callback (cons 'error errmsg) nil)))
                        (when ensure
                          (funcall ensure))
                        (ignore-errors
@@ -1040,23 +1049,28 @@ purposes."
           (apheleia--execute-formatter-process
            :ctx ctx
            :callback
-           (lambda (stdout)
-             (when-let
-                 ((output-fname (apheleia-formatter--output-fname ctx)))
-               ;; Load output-fname contents into the stdout buffer.
-               (with-current-buffer stdout
-                 (erase-buffer)
-                 (insert-file-contents-literally output-fname)))
-             (funcall callback stdout))
+           (lambda (err stdout)
+             (if err
+                 (funcall callback err stdout)
+               (when-let
+                   ((output-fname (apheleia-formatter--output-fname ctx)))
+                 ;; Load output-fname contents into the stdout buffer.
+                 (with-current-buffer stdout
+                   (erase-buffer)
+                   (insert-file-contents-literally output-fname)))
+               (funcall callback nil stdout)))
            :ensure
            (lambda ()
              (dolist (fname (list (apheleia-formatter--input-fname ctx)
                                   (apheleia-formatter--output-fname ctx)))
                (when fname
                  (ignore-errors (delete-file fname))))))
-        (apheleia--log
-         'process
-         "Could not find executable for formatter %s, skipping" formatter)))))
+        (let ((errmsg
+               (format
+                "Could not find executable for formatter %s, skipping"
+                formatter)))
+          (apheleia--log 'process "%s" errmsg)
+          (funcall callback (cons 'error errmsg) nil))))))
 
 (defun apheleia--run-formatter-function
     (func buffer remote callback stdin formatter)
@@ -1087,11 +1101,14 @@ being run, for diagnostic purposes."
                :scratch scratch
                ;; Name of the current formatter symbol, e.g. `black'.
                :formatter formatter
-               ;; Callback after successfully formatting.
+               ;; Callback. Should pass an error value (cons of symbol
+               ;; and data, like for `signal') or nil. For backwards
+               ;; compatibility it can also invoke only on success,
+               ;; with no args.
                :callback
-               (lambda ()
+               (lambda (&optional err)
                  (unwind-protect
-                     (funcall callback scratch)
+                     (funcall callback err (when (not err) scratch))
                    (kill-buffer scratch)))
                ;; The remote part of the buffers file-name or directory.
                :remote remote
@@ -1123,20 +1140,25 @@ For more implementation detail, see
       (indent-region (point-min) (point-max)))
     (funcall callback)))
 
-(defun apheleia--run-formatters
+(cl-defun apheleia--run-formatters
     (formatters buffer remote callback &optional stdin)
   "Run one or more code formatters on the current buffer.
 FORMATTERS is a list of symbols that appear as keys in
 `apheleia-formatters'. BUFFER is the `current-buffer' when this
-function was first called. Once all the formatters in COMMANDS
-finish successfully then invoke CALLBACK with one argument, a
-buffer containing the output of all the formatters. REMOTE asserts
-whether the buffer being formatted is on a remote machine or the
-current machine. It should be the output of `file-remote-p' on the
-current variable `buffer-file-name'. REMOTE is the remote part of the
-original buffers file-name or directory'. It's used alongside
-`apheleia-remote-algorithm' to determine where the formatter process
-and any temporary files it may need should be placed.
+function was first called.
+
+CALLBACK is always invoked unless there is a synchronous nonlocal
+exit, the first argument is nil or an error. In the case of no
+error, the second argument is a buffer containing the output of
+all the formatters, otherwise it is nil.
+
+REMOTE asserts whether the buffer being formatted is on a remote
+machine or the current machine. It should be the output of
+`file-remote-p' on the current variable `buffer-file-name'.
+REMOTE is the remote part of the original buffers file-name or
+directory'. It's used alongside `apheleia-remote-algorithm' to
+determine where the formatter process and any temporary files it
+may need should be placed.
 
 STDIN is a buffer containing the standard input for the first
 formatter in COMMANDS. This should not be supplied by the caller
@@ -1160,15 +1182,20 @@ function: %s" command)))
      command
      buffer
      remote
-     (lambda (stdout)
-       (unless (string-empty-p (with-current-buffer stdout (buffer-string)))
-         (if (cdr formatters)
-             ;; Forward current stdout to remaining formatters, passing along
-             ;; the current callback and using the current formatters output
-             ;; as stdin.
-             (apheleia--run-formatters
-              (cdr formatters) buffer remote callback stdout)
-           (funcall callback stdout))))
+     (lambda (err stdout)
+       (if err
+           (funcall callback err stdout)
+         (condition-case-unless-debug err
+             (unless (string-empty-p
+                      (with-current-buffer stdout (buffer-string)))
+               (if (cdr formatters)
+                   ;; Forward current stdout to remaining formatters,
+                   ;; passing along the current callback and using the
+                   ;; current formatters output as stdin.
+                   (apheleia--run-formatters
+                    (cdr formatters) buffer remote callback stdout)
+                 (funcall callback nil stdout)))
+           (error (funcall callback err nil)))))
      stdin
      (car formatters))))
 
diff --git a/apheleia-log.el b/apheleia-log.el
index 3547011..77b5177 100644
--- a/apheleia-log.el
+++ b/apheleia-log.el
@@ -145,11 +145,14 @@ callables by accident."
             (error (setq body (format "Got error formatting log line %S: %s"
                                       message
                                       (error-message-string err)))))
-          (insert
-           (format
-            "%s <%S>: %s\n"
-            (format-time-string "%Y-%m-%d %H:%M:%S.%3N" (current-time))
-            category body)))))))
+          (let ((msg
+                 (format
+                  "%s <%S>: %s"
+                  (format-time-string "%Y-%m-%d %H:%M:%S.%3N" (current-time))
+                  category body)))
+            (insert msg "\n")
+            (when noninteractive
+              (message "%s" msg))))))))
 
 (provide 'apheleia-log)
 
diff --git a/apheleia.el b/apheleia.el
index 3778c8a..a419f47 100644
--- a/apheleia.el
+++ b/apheleia.el
@@ -53,8 +53,23 @@
              (eq apheleia-remote-algorithm 'cancel))
     "Apheleia refused to run formatter due to `apheleia-remote-algorithm'"))
 
+(defmacro apheleia--with-on-error (on-error &rest body)
+  "Call ON-ERROR with an error if BODY throws an error.
+Return the error in that case, instead of throwing it. If
+ON-ERROR is nil, instead act just like `progn'."
+  (declare (indent 1))
+  (let ((err-sym (make-symbol "err"))
+        (on-error-sym (make-symbol "on-error")))
+    `(let ((,on-error-sym ,on-error))
+       (if ,on-error-sym
+           (condition-case-unless-debug ,err-sym
+               (progn ,@body)
+             (error (funcall ,on-error-sym ,err-sym)))
+         (progn ,@body)))))
+
 ;;;###autoload
-(defun apheleia-format-buffer (formatter &optional callback)
+(cl-defun apheleia-format-buffer
+    (formatter &optional success-callback &key callback)
   "Run code formatter asynchronously on current buffer, preserving point.
 
 FORMATTER is a symbol appearing as a key in
@@ -74,7 +89,13 @@ however, the operation is aborted.
 
 If the formatter actually finishes running and the buffer is
 successfully updated (even if the formatter has not made any
-changes), CALLBACK, if provided, is invoked with no arguments."
+changes), SUCCESS-CALLBACK, if provided, is invoked with no
+arguments.
+
+If provided, CALLBACK is invoked unconditionally (unless there is
+a synchronous nonlocal exit) with a plist. Callback function must
+accept unknown keywords. At present only `:error' is included,
+this is either an error or nil."
   (interactive (progn
                  (when-let ((err (apheleia--disallowed-p)))
                    (user-error err))
@@ -82,80 +103,101 @@ changes), CALLBACK, if provided, is invoked with no arguments."
                         (if current-prefix-arg
                             'prompt
                           'interactive)))))
-  (apheleia--log
-   'format-buffer
-   "Invoking apheleia-format-buffer on %S with formatter %S"
-   (current-buffer)
-   formatter)
-  (let ((formatters (apheleia--ensure-list formatter)))
-    ;; Check for this error ahead of time so we don't have to deal
-    ;; with it anywhere in the internal machinery of Apheleia.
-    (dolist (formatter formatters)
-      (unless (alist-get formatter apheleia-formatters)
-        (user-error
-         "No such formatter defined in `apheleia-formatters': %S"
-         formatter)))
-    ;; Fail silently if disallowed, since we don't want to throw an
-    ;; error on `post-command-hook'. We already took care of throwing
-    ;; `user-error' on interactive usage above.
-    (if-let ((err (apheleia--disallowed-p)))
-        (apheleia--log
-         'format-buffer
-         "Aborting in %S due to apheleia--disallowed-p: %s"
-         (buffer-name (current-buffer))
-         err)
-      ;; It's important to store the saved buffer hash in a lexical
-      ;; variable rather than a dynamic (global) one, else multiple
-      ;; concurrent invocations of `apheleia-format-buffer' can
-      ;; overwrite each other, and get the wrong results about whether
-      ;; the buffer was actually modified since the formatting
-      ;; operation started, leading to data loss.
-      ;;
-      ;; https://github.com/radian-software/apheleia/issues/226
-      (let ((saved-buffer-hash (apheleia--buffer-hash)))
-        (let ((cur-buffer (current-buffer))
-              (remote (file-remote-p (or buffer-file-name
-                                         default-directory))))
-          (apheleia--run-formatters
-           formatters
-           cur-buffer
-           remote
-           (lambda (formatted-buffer)
-             (if (not (buffer-live-p cur-buffer))
-                 (apheleia--log
-                  'format-buffer
-                  "Aborting in %S because buffer has died"
-                  (buffer-name cur-buffer))
-               (with-current-buffer cur-buffer
-                 ;; Short-circuit.
-                 (if (not (equal saved-buffer-hash (apheleia--buffer-hash)))
-                     (apheleia--log
-                      'format-buffer
-                      "Aborting in %S because contents have changed"
-                      (buffer-name cur-buffer))
-                   (apheleia--create-rcs-patch
-                    cur-buffer formatted-buffer remote
-                    (lambda (patch-buffer)
-                      (when (buffer-live-p cur-buffer)
-                        (with-current-buffer cur-buffer
-                          (if (not (equal
-                                    saved-buffer-hash
-                                    (apheleia--buffer-hash)))
-                              (apheleia--log
-                               'format-buffer
-                               "Aborting in %S because contents have changed"
-                               (buffer-name cur-buffer))
-                            (apheleia--apply-rcs-patch
-                             (current-buffer) patch-buffer)
-                            (if (not callback)
-                                (apheleia--log
-                                 'format-buffer
-                                 (concat
-                                  "Skipping callback because "
-                                  "none was provided"))
-                              (apheleia--log
-                               'format-buffer "Invoking callback")
-                              (funcall callback)))))))))))))))))
+  (let ((callback
+         (lambda (err)
+           (unless (listp err)
+             (setq err (cons 'error err)))
+           (unless err
+             (when success-callback
+               (funcall success-callback)))
+           (when callback
+             (funcall callback :error err)))))
+    (apheleia--log
+     'format-buffer
+     "Invoking apheleia-format-buffer on %S with formatter %S"
+     (current-buffer)
+     formatter)
+    (let ((formatters (apheleia--ensure-list formatter)))
+      ;; Check for this error ahead of time so we don't have to deal
+      ;; with it anywhere in the internal machinery of Apheleia.
+      (dolist (formatter formatters)
+        (unless (alist-get formatter apheleia-formatters)
+          (user-error
+           "No such formatter defined in `apheleia-formatters': %S"
+           formatter)))
+      ;; Fail silently if disallowed, since we don't want to throw an
+      ;; error on `post-command-hook'. We already took care of throwing
+      ;; `user-error' on interactive usage above.
+      (if-let ((err (apheleia--disallowed-p)))
+          (progn
+            (apheleia--log
+             'format-buffer
+             "Aborting in %S due to apheleia--disallowed-p: %s"
+             (buffer-name (current-buffer))
+             err)
+            (when callback
+              (funcall callback err)))
+        ;; It's important to store the saved buffer hash in a lexical
+        ;; variable rather than a dynamic (global) one, else multiple
+        ;; concurrent invocations of `apheleia-format-buffer' can
+        ;; overwrite each other, and get the wrong results about whether
+        ;; the buffer was actually modified since the formatting
+        ;; operation started, leading to data loss.
+        ;;
+        ;; https://github.com/radian-software/apheleia/issues/226
+        (let ((saved-buffer-hash (apheleia--buffer-hash)))
+          (let ((cur-buffer (current-buffer))
+                (remote (file-remote-p (or buffer-file-name
+                                           default-directory))))
+            (apheleia--run-formatters
+             formatters
+             cur-buffer
+             remote
+             (lambda (err formatted-buffer)
+               (if err
+                   (funcall callback err)
+                 (apheleia--with-on-error callback
+                   (if (not (buffer-live-p cur-buffer))
+                       (progn
+                         (apheleia--log
+                          'format-buffer
+                          "Aborting in %S because buffer has died"
+                          (buffer-name cur-buffer))
+                         (funcall callback "Buffer has died"))
+                     (with-current-buffer cur-buffer
+                       ;; Short-circuit.
+                       (if (not (equal
+                                 saved-buffer-hash (apheleia--buffer-hash)))
+                           (progn
+                             (apheleia--log
+                              'format-buffer
+                              "Aborting in %S because contents have changed"
+                              (buffer-name cur-buffer))
+                             (funcall callback "Contents have changed"))
+                         (apheleia--create-rcs-patch
+                          cur-buffer formatted-buffer remote
+                          (lambda (err patch-buffer)
+                            (if err
+                                (funcall callback err)
+                              (apheleia--with-on-error callback
+                                (when (buffer-live-p cur-buffer)
+                                  (with-current-buffer cur-buffer
+                                    (if (not (equal
+                                              saved-buffer-hash
+                                              (apheleia--buffer-hash)))
+                                        (progn
+                                          (apheleia--log
+                                           'format-buffer
+                                           (concat
+                                            "Aborting in %S because "
+                                            "contents have changed")
+                                           (buffer-name cur-buffer))
+                                          (funcall
+                                           callback "Contents have changed"))
+                                      (apheleia--apply-rcs-patch
+                                       (current-buffer) patch-buffer)
+                                      (funcall
+                                       callback nil)))))))))))))))))))))
 
 (defcustom apheleia-post-format-hook nil
   "Normal hook run after Apheleia formats a buffer successfully."
diff --git a/test/integration/apheleia-it.el b/test/integration/apheleia-it.el
new file mode 100644
index 0000000..7c5ee3d
--- /dev/null
+++ b/test/integration/apheleia-it.el
@@ -0,0 +1,228 @@
+;; -*- lexical-binding: t -*-
+
+;; `apheleia-it' - short for `apheleia-integration-tests'. The
+;; functions in here are not part of the public interface of Apheleia
+;; and breaking changes may occur at any time.
+
+(require 'apheleia)
+
+(require 'cl-lib)
+
+(defvar apheleia-it-mode-keymap
+  (let ((map (make-sparse-keymap)))
+    (prog1 map
+      (define-key map (kbd "q") #'quit-window)))
+  "Keymap for use in `apheleia-it-mode'.")
+
+(define-minor-mode apheleia-it-mode
+  "Minor mode to add some keybindings in test result buffers."
+  :keymap apheleia-it-mode-keymap)
+
+(defvar apheleia-it-tests nil
+  "List of integration tests, an alist.")
+(setq apheleia-it-tests nil)
+
+(cl-defmacro apheleia-it-deftest
+    (name desc &rest kws &key scripts formatters steps)
+  "Declare a integration test."
+  (declare (indent defun) (doc-string 2))
+  (ignore scripts formatters steps)
+  `(progn
+     (when (alist-get ',name apheleia-it-tests)
+       (message "Overwriting existing test: %S" ',name))
+     (setf (alist-get ',name apheleia-it-tests) (list :desc ,desc ,@kws))))
+
+(defvar apheleia-it-workdir
+  (file-name-directory (or load-file-name buffer-file-name))
+  "Directory that this variable is defined in.")
+
+(defvar apheleia-it-timers nil
+  "List of timers that should be canceled or finished before exit.")
+
+(defun apheleia-it-run-with-timer (secs function &rest args)
+  "Like `run-with-timer' but delays Emacs exit until done or canceled."
+  (let ((timer (apply #'run-with-timer secs nil function args)))
+    (prog1 timer
+      (push timer apheleia-it-timers))))
+
+(defun apheleia-it-timers-active-p ()
+  "Non-nil if there are any active Apheleia timers for tests.
+This may mutate `apheleia-it-timers' to cleanup expired timers."
+  (cl-block nil
+    (while apheleia-it-timers
+      (if (memq (car apheleia-it-timers) timer-list)
+          (cl-return t)
+        (setq apheleia-it-timers (cdr apheleia-it-timers))))))
+
+(defun apheleia-it--run-test-steps (steps bindings callback)
+  "Run STEPS from defined integration test.
+This is a list that can appear in `:steps'. For supported steps,
+see the implementation below, or example tests. BINDINGS is a
+`let'-style list of lexical bindings that will be available for
+`eval' steps. CALLBACK will be invoked, with nil or an error,
+after the steps are run. This could be synchronous or
+asynchronous."
+  (apheleia--log
+   'test "Running test step %s"
+   (replace-regexp-in-string
+    "\n" "\\n" (format "%S" (car steps)) nil 'literal))
+  (condition-case-unless-debug err
+      (pcase steps
+        (`nil (funcall callback nil))
+        (`((with-callback ,callback-sym . ,body) . ,rest)
+         (let* ((callback-called nil)
+                (timeout-timer nil)
+                (wrapped-callback
+                 (lambda (err)
+                   (when (timerp timeout-timer)
+                     (cancel-timer timeout-timer))
+                   (unless callback-called
+                     (setq callback-called t)
+                     (if err
+                         (funcall callback err)
+                       (apheleia-it--run-test-steps
+                        rest bindings callback))))))
+           (setq timeout-timer
+                 (apheleia-it-run-with-timer
+                  3 wrapped-callback
+                  (cons 'error (format
+                                "Callback not invoked within timeout for %S"
+                                body))))
+           (apheleia-it--run-test-steps
+            body
+            (cons
+             (cons callback-sym
+                   wrapped-callback)
+             bindings)
+            #'ignore)))
+        (`((eval ,form))
+         (eval form bindings)
+         (funcall callback nil))
+        (`((insert ,str) . ,rest)
+         (erase-buffer)
+         (let ((p (string-match-p "|" str)))
+           (insert (replace-regexp-in-string "|" "" str nil 'literal))
+           (goto-char p))
+         (apheleia-it--run-test-steps rest bindings callback))
+        (`((expect ,str) . ,rest)
+         (cl-assert (eq (point) (string-match-p "|" str)))
+         (cl-assert
+          (string=
+           (buffer-string)
+           (replace-regexp-in-string "|" "" str nil 'literal)))
+         (funcall callback nil))
+        (_ (error "Malformed test step `%S'" (car steps))))
+    (error (funcall callback err))))
+
+(defun apheleia-it-run-test (name callback)
+  "Run a single integration test. Invoke CALLBACK with nil or an error."
+  (interactive
+   (list
+    (intern
+     (completing-read
+      "Run test: "
+      (mapcar #'symbol-name (map-keys apheleia-it-tests))))
+    (lambda (err)
+      (if err
+          (signal (car err) (cdr err))
+        (message "Test passed" (length apheleia-it-tests))))))
+  (message "Running test %S" name)
+  (condition-case-unless-debug err
+      (let* ((test (alist-get name apheleia-it-tests))
+             (bufname (format " *apheleia-it test %S*" name))
+             (result nil))
+        (unless (plist-get test :steps)
+          (user-error "Incomplete test: %S" name))
+        (when (get-buffer bufname)
+          (kill-buffer bufname))
+        (pop-to-buffer bufname)
+        (setq-local default-directory apheleia-it-workdir)
+        (fundamental-mode)
+        (apheleia-it-mode +1)
+        (ignore-errors
+          (delete-directory ".tmp" 'recursive))
+        (make-directory ".tmp")
+        (dolist (script (plist-get test :scripts))
+          (with-temp-buffer
+            (insert (cdr script))
+            (let ((fname (expand-file-name (format ".tmp/%s" (car script)))))
+              (write-file fname)
+              (chmod fname #o755))))
+        (setq-local exec-path (cons (expand-file-name ".tmp") exec-path))
+        (setq-local apheleia-formatters (plist-get test :formatters))
+        (apheleia-it--run-test-steps (plist-get test :steps) nil callback))
+    (error (funcall callback err))))
+
+(defun apheleia-it-run-tests (names callback)
+  "Run multiple integration tests. Stop on error.
+Invoke CALLBACK with nil or an error."
+  (if names
+      (apheleia-it-run-test
+       (car names)
+       (lambda (err)
+         (if err
+             (funcall callback err)
+           (apheleia-it-run-tests (cdr names) callback))))
+    (funcall callback nil)))
+
+(defun apheleia-it-run-all-tests ()
+  "Run all the integration tests until a failure is encountered."
+  (interactive)
+  (apheleia-it-run-tests
+   (nreverse (map-keys apheleia-it-tests))
+   (lambda (err)
+     (if err
+         (signal (car err) (cdr err))
+       (message "All %d tests passed" (length apheleia-it-tests)))))
+  (when noninteractive
+    (while (apheleia-it-timers-active-p)
+      (sit-for 0.5))))
+
+(cl-defun apheleia-it-script (&key allowed-inputs)
+  "Return text of a bash script to act as a mock formatter.
+Keyword arguments control the behavior. ALLOWED-INPUTS is an
+alist of inputs that are allowed to be passed to the formatter,
+along with the outputs that is will return. Any other input will
+generate an error."
+  (concat
+   "#!/usr/bin/env bash
+input=\"$(cat; echo x)\"
+input=\"${input%x}\"
+"
+   (mapcan
+    (lambda (link)
+      (cl-destructuring-bind (input . output) link
+        (format
+         "expected_input=%s
+expected_output=%s
+if [[ \"${input}\" == \"${expected_input}\" ]]; then
+    printf '%%s' \"${expected_output}\"
+    exit 0
+fi
+"
+         (shell-quote-argument input)
+         (shell-quote-argument output))))
+    allowed-inputs)
+   "echo >&2 'formatter got unexpected input'
+echo >&2 'received input follows:'
+echo \"${input}\" | sed 's/^/| /' >&2
+exit 1
+"))
+
+(apheleia-it-deftest basic-functionality
+  "Running `apheleia-format-buffer' does formatting"
+  :scripts `(("apheleia-it" .
+              ,(apheleia-it-script
+                :allowed-inputs
+                '(("The quick brown fox jumped over the lazy dog\n" .
+                   "The slow brown fox jumped over the studious dog\n")))))
+  :formatters '((apheleia-it . ("apheleia-it")))
+  :steps '((insert "The quick brown fox jum|ped over the lazy dog\n")
+           (with-callback
+            callback
+            (eval (apheleia-format-buffer
+                   'apheleia-it nil
+                   :callback
+                   (lambda (&rest props)
+                     (funcall callback (plist-get props :error))))))
+           (expect "The slow brown fox jum|ped over the studious dog\n")))