[WIP] Add make job to refresh integration tests

Makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2018 Flycheck contributors
+# Copyright (c) 2018, 2020 Flycheck contributors
 # Copyright (c) 2012-2016 Sebastian Wiesner and Flycheck contributors
 
 # This program is free software: you can redistribute it and/or modify it under
@@ -50,6 +50,10 @@ HAVE_OPTIPNG := $(shell sh -c "command -v $(OPTIPNG)")
 RUNTEST = $(RUNEMACS) --load test/flycheck-test --load test/run.el \
  -f flycheck-run-tests-main
 
+RUN_REGEN = $(RUNEMACS) --load test/flycheck-test --load test/run.el \
+ --load test/setup-regen.el \
+ -f flycheck-run-tests-main
+
 # Export Emacs to goals, mainly for CASK
 CASK_EMACS = $(EMACS)
 export EMACS
@@ -131,6 +135,10 @@ unit: compile
 integ: compile
  $(RUNTEST) '(and (tag external-tool) $(SELECTOR))'
 
+.PHONY: integ-regen
+integ-regen: compile
+ $(RUN_REGEN) '(and (tag external-tool) $(SELECTOR))'
+
 .PHONY: images
 images: $(IMGS)
 
@@ -139,23 +147,24 @@ help:
  @echo 'Run `make init` first to install and update all local dependencies.'
  @echo ''
  @echo 'Available targets:'
- @echo ' init: Initialise the project. RUN FIRST!'
- @echo ' check: Check all Emacs Lisp sources (needs Emacs 25)'
- @echo ' compile: Byte-compile Emacs Lisp sources'
- @echo ' format: Format all Emacs Lisp sources'
- @echo ' specs: Run all buttercup specs for Flycheck'
- @echo ' unit: Run all ERT unit tests for Flycheck (legacy)'
- @echo ' integ: Run all integration tests for Flycheck'
- @echo ' images: Generate PNG images from SVG sources'
- @echo ' clean: Clean compiled files'
- @echo ' purge: Clean everything'
+ @echo ' init: Initialise the project. RUN FIRST!'
+ @echo ' check: Check all Emacs Lisp sources (needs Emacs 25)'
+ @echo ' compile: Byte-compile Emacs Lisp sources'
+ @echo ' format: Format all Emacs Lisp sources'
+ @echo ' specs: Run all buttercup specs for Flycheck'
+ @echo ' unit: Run all ERT unit tests for Flycheck (legacy)'
+ @echo ' integ: Run all integration tests for Flycheck'
+ @echo ' integ-regen: Regenerate expected integration tests results'
+ @echo ' images: Generate PNG images from SVG sources'
+ @echo ' clean: Clean compiled files'
+ @echo ' purge: Clean everything'
  @echo ''
  @echo 'Available make variables:'
- @echo ' PATTERN: A regular expression matching spec names to run with `specs`'
- @echo ' SELECTOR: An ERT selector expression for `unit` and `integ`'
- @echo ' LANGUAGE: The name of a language for `integ`. Overrides `SELECTOR`'
+ @echo ' PATTERN:  A regular expression matching spec names to run with `specs`'
+ @echo ' SELECTOR:  An ERT selector expression for `unit` and `integ`'
+ @echo ' LANGUAGE:  The name of a language for `integ`. Overrides `SELECTOR`'
  @echo ' EMACSOPTS: Additional options to pass to `emacs`'
- @echo ' EMACS: The path or name of the Emacs to use for tests and compilation'
+ @echo ' EMACS:  The path or name of the Emacs to use for tests and compilation'
  @echo ''
  @echo 'Available programs:'
  @echo ' $(CASK): $(if $(HAVE_CASK),yes,no)'

test/setup-regen.el

@@ -0,0 +1,109 @@
+;;; package --- Summary
+;;; Commentary:
+;;; Code:
+
+(require 'flycheck-ert)
+
+(defun flycheck-ert-should-syntax-check (resource-file modes &rest errors)
+ "Test a syntax check in RESOURCE-FILE with MODES.
+
+RESOURCE-FILE is the file to check. MODES is a single major mode
+symbol or a list thereof, specifying the major modes to syntax
+check with. If more than one major mode is specified, the test
+is run for each mode separately, so if you give three major
+modes, the entire test will run three times. ERRORS is the list
+of expected errors, as in `flycheck-ert-should-errors'. If
+omitted, the syntax check must not emit any errors. The errors
+are cleared after each test.
+
+The syntax checker is selected via standard syntax checker
+selection. To test a specific checker, you need to set
+`flycheck-checker' or `flycheck-disabled-checkers' accordingly
+before using this predicate, depending on whether you want to use
+manual or automatic checker selection.
+
+During the syntax check, configuration files of syntax checkers
+are also searched in the `config-files' sub-directory of the
+resource directory."
+ (when (symbolp modes)
+ (setq modes (list modes)))
+ (dolist (mode modes)
+ (unless (fboundp mode)
+ (ert-skip (format "%S missing" mode)))
+ (flycheck-ert-with-resource-buffer resource-file
+ (funcall mode)
+ ;; Load safe file-local variables because some tests depend on them
+ (let ((enable-local-variables :safe)
+ ;; Disable all hooks at this place, to prevent 3rd party packages
+ ;; from interfering
+ (hack-local-variables-hook))
+ (hack-local-variables))
+ ;; Configure config file locating for unit tests
+ (let ((process-hook-called 0))
+ (add-hook 'flycheck-process-error-functions
+ (lambda (_err)
+ (setq process-hook-called (1+ process-hook-called))
+ nil)
+ nil :local)
+ (add-hook 'flycheck-status-changed-functions
+ (lambda (status)
+ (when (eq status 'suspicious)
+ (signal 'flycheck-ert-suspicious-checker nil))))
+ (flycheck-ert-buffer-sync)
+ (apply #'flycheck-ert-regen-errors errors))
+ (flycheck-ert-ensure-clear))))
+
+(defun flycheck-ert-regen-errors (&rest expected-errors)
+ "Replace errors in flycheck-test.el.
+
+Find EXPECTED-ERRORS in flycheck-test.el and replace them by
+flycheck-current-errors."
+ (let ((test-filename (expand-file-name
+ "flycheck-test.el"
+ (locate-dominating-file default-directory "flycheck-test.el")))
+ (expected-errors-re-string
+ (mapconcat
+ (lambda (err)
+ (replace-regexp-in-string " +" "[\n ]+"
+ (regexp-quote (format "'%S" err))
+ 'fixedcase 'literal))
+ expected-errors "[\n ]+"))
+ (replacement-errors
+ (flycheck-ert-errors-to-string flycheck-current-errors)))
+ ;;(message "%S" expected-errors-re-string)
+ (with-current-buffer (find-file test-filename)
+ (save-excursion
+ (goto-char (point-min))
+ (when (re-search-forward expected-errors-re-string nil 'noerror)
+ (let ((replace-start (match-beginning 0))
+ (replace-end (match-end 0)))
+ (delete-region replace-start replace-end)
+ (insert replacement-errors)
+ ;;(message "Replaced %s to %s" replace-start replace-end)
+ (save-buffer)))))))
+
+(defun flycheck-ert-errors-to-string (errors)
+ "Convert ERRORS objects to string."
+ (let ((group-counter 0))
+ (mapconcat
+ (lambda (err)
+ (format "'%S"
+ ;; TODO: do not add properties that are nil
+ `(,(flycheck-error-line err)
+ ,(flycheck-error-column err)
+ ,(flycheck-error-level err)
+ ,(flycheck-error-message err)
+ :checker ,(flycheck-error-checker err)
+ :id ,(flycheck-error-id err)
+ :group ,(when (flycheck-error-group err)
+ (unless (get (flycheck-error-group err) 'id)
+ (put (flycheck-error-group err) 'id (setq group-counter (1+ group-counter))))
+ (get (flycheck-error-group err) 'id))
+ :end-line ,(flycheck-error-end-line err)
+ :end-column ,(flycheck-error-end-column err)
+ )))
+ errors "\n")))
+
+(provide 'setup-regen)
+
+;;; setup-regen ends here