Refactor test system

Should allow easier testing of layers, not just core

.travis.yml

@@ -10,4 +10,4 @@ before_install:
   sudo apt-get -qqy install emacs-snapshot;
 
 script:
-  ./travis/run_build.sh
+  ./travis/run_build.sh

core/Makefile

@@ -1,43 +0,0 @@
-## Makefile --- Spacemacs Core File
-##
-## Copyright (c) 2012-2014 Sylvain Benner
-## Copyright (c) 2014-2015 Sylvain Benner & Contributors
-##
-## Author: Sylvain Benner <sylvain.benner@gmail.com>
-## URL: https://github.com/syl20bnr/spacemacs
-##
-## This file is not part of GNU Emacs.
-##
-## License: GPLv3
-EMACS_DIR=~/.emacs.d/
-TEST_DIR=$(EMACS_DIR)core/tests/
-
-all: test
-
-test: unit_tests func_tests
-
-unit_tests:
-	@echo "-----------------------------------------------------------------"
-	@echo "Unit Tests"
-	@echo "-----------------------------------------------------------------"
-	@emacs -batch -l ert  \
-    -l $(EMACS_DIR)core/core-load-paths.el  \
-    -l $(TEST_DIR)core-evilify-keymap-utest.el  \
-    -l $(TEST_DIR)core-configuration-layer-utest.el \
-    -l $(TEST_DIR)core-funcs-utest.el  \
-    -f ert-run-tests-batch-and-exit
-
-func_tests:
-	@echo "-----------------------------------------------------------------"
-	@echo "Functional Tests"
-	@echo "-----------------------------------------------------------------"
-	@emacs -batch -l ert  \
-    -l $(EMACS_DIR)core/core-load-paths.el  \
-    -l $(TEST_DIR)core-evilify-keymap-ftest.el  \
-    -l $(TEST_DIR)core-spacemacs-ftest.el  \
-    -l $(TEST_DIR)core-spacemacs-buffer-ftest.el  \
-    -l $(TEST_DIR)core-release-management-ftest.el  \
-    -f ert-run-tests-batch-and-exit
-
-
-.PHONY: test unit_tests func_test

core/core-load-paths.el

@@ -39,6 +39,9 @@
 (defconst spacemacs-docs-directory
   (expand-file-name (concat user-emacs-directory "doc/"))
   "Spacemacs documentation directory.")
+(defconst spacemacs-test-directory
+  (expand-file-name (concat user-emacs-directory "tests/"))
+  "Spacemacs tests directory.")
 
 (defconst user-home-directory
   (expand-file-name "~/")

doc/CONTRIBUTE.org

@@ -7,6 +7,7 @@
          - [[#for-complex-pull-requests][For complex pull requests:]]
          - [[#getting-help][Getting Help]]
      - [[#submitting-a-configuration-layer][Submitting a configuration layer]]
+     - [[#testing][Testing]]
      - [[#submitting-a-banner][Submitting a banner]]
      - [[#credits][Credits]]
          - [[#license][License]]
@@ -61,6 +62,46 @@ It is recommended to join a =README.org= file with your layer:
   - if a logo exists for the layer you can add it at the top of the =README.org=
     before the TOC. The maximum recommended height is 200 pixels.
 
+** Testing
+Tests live in the =tests= folder, with a folder structure corresponding to the
+rest of the repository.
+
+To run tests locally, navigate to the relevant subfolder and run =make=. The
+tests will be run with your own personal dotfile, so you should not expect tests
+to succeed if you have personal configuration that will make them fail (such as
+not having enabled a layer).
+
+Spacemacs uses Travis CI to perform more comprehensive testing, where each
+testable layer is enabled in turn.
+
+To add tests for a layer, do the following:
+
+1. Create a subfolder of =tests= corresponding to the layer you want to test.
+2. Write a file called =dotspacemacs.el= in that folder. It should be a minimal
+   dotfile that enables the layer in question (and other layers it may depend
+   on).
+3. Write a number of files with tests. Please try to separate unit and
+   functional tests. Look at existing tests for clues.
+4. Write a =Makefile= in that folder. It should define three variables.
+   - =LOAD_FILES= :: a list of additional files to load before testing (relative
+                     to the root Spacemacs folder). This should typically be
+ =init.el=.
+   - =UNIT_TEST_FILES= :: a list of unit test files in the current folder.
+   - =FUNC_TEST_FILES= :: a list of functional test files in the current folder.
+   See existing tests for examples.
+   #+begin_src makefile
+   TEST_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+
+   LOAD_FILES = ...
+   UNIT_TEST_FILES = ...
+   FUNC_TEST_FILES = ...
+
+   include ../../spacemacs.mk
+   #+end_src
+5. Add the new test to list of tests in =travis/run_build.sh=.
+
+Spacemacs is lacking tests, so contributions are welcome.
+
 ** Submitting a banner
 The startup banner is by default randomly chosen among a pool of banners
 each time =Spacemacs= starts. Banners are located in directory

spacemacs.mk

@@ -0,0 +1,42 @@
+## Makefile --- Spacemacs master makefile
+##
+## Copyright (c) 2012-2014 Sylvain Benner
+## Copyright (c) 2014-2015 Sylvain Benner & Contributors
+##
+## Author: Sylvain Benner <sylvain.benner@gmail.com>
+## URL: https://github.com/syl20bnr/spacemacs
+##
+## This file is not part of GNU Emacs.
+##
+## License: GPLv3
+
+EMACS_DIR = $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+TEST_NAME = `basename $(TEST_DIR) | tr a-z A-Z`
+
+all: test
+
+test: unit_tests func_tests
+
+ifneq ($(strip $(UNIT_TEST_FILES)),)
+unit_tests:
+	@echo "================================================================="
+	@echo "UNIT TESTS FOR $(TEST_NAME)"
+	@echo "================================================================="
+	@emacs -batch -l ert \
+    $(addprefix -l $(EMACS_DIR)/, $(LOAD_FILES)) \
+    $(addprefix -l $(TEST_DIR)/, $(UNIT_TEST_FILES)) \
+    -f ert-run-tests-batch-and-exit
+endif
+
+ifneq ($(strip $(FUNC_TEST_FILES)),)
+func_tests:
+	@echo "================================================================="
+	@echo "FUNCTIONAL TESTS FOR $(TEST_NAME)"
+	@echo "================================================================="
+	@emacs -batch -l ert \
+    $(addprefix -l $(EMACS_DIR)/, $(LOAD_FILES)) \
+    $(addprefix -l $(TEST_DIR)/, $(FUNC_TEST_FILES)) \
+    -f ert-run-tests-batch-and-exit
+endif
+
+.PHONY: test unit_tests func_tests

tests/core/Makefile

@@ -0,0 +1,26 @@
+## Makefile --- Spacemacs Core
+##
+## Copyright (c) 2012-2014 Sylvain Benner
+## Copyright (c) 2014-2015 Sylvain Benner & Contributors
+##
+## Author: Sylvain Benner <sylvain.benner@gmail.com>
+## URL: https://github.com/syl20bnr/spacemacs
+##
+## This file is not part of GNU Emacs.
+##
+## License: GPLv3
+
+TEST_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+
+LOAD_FILES = core/core-load-paths.el
+UNIT_TEST_FILES = \
+  core-evilify-keymap-utest.el \
+  core-configuration-layer-utest.el \
+  core-funcs-utest.el
+FUNC_TEST_FILES = \
+  core-evilify-keymap-ftest.el \
+  core-spacemacs-ftest.el \
+  core-spacemacs-buffer-ftest.el \
+  core-release-management-ftest.el
+
+include ../../spacemacs.mk

tests/core/core-spacemacs-buffer-ftest.el

@@ -104,7 +104,7 @@
 
 (ert-deftest test-render-framed-text--file-caption-and-padding ()
   (should (equal (spacemacs//render-framed-text
-                  (concat spacemacs-core-directory "tests/data/framed-text.txt")
+                  (concat spacemacs-test-directory "core/data/framed-text.txt")
                   62 "Caption" 4)
                  "╭─Caption────────────────────────────────────────────────────╮
 │                                                            │

travis/run_build.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
 ## run_build.sh --- Travis CI File for Spacemacs
 ##
 ## Copyright (c) 2012-2014 Sylvain Benner
@@ -10,6 +10,26 @@
 ## This file is not part of GNU Emacs.
 ##
 ## License: GPLv3
-ln -s `pwd` /home/travis/.emacs.d
-cd ~/.emacs.d/core
-make test
+
+tests=("core" "spacemacs")
+
+if [ $USER != "travis" ]; then
+    echo "This script is not designed to run locally."
+    echo "Instead, navigate to the appropriate test folder and run make there instead."
+    exit 1
+fi
+
+ln -s `pwd` ~/.emacs.d
+
+for test in "${tests[@]}"; do
+    rm -rf ~/.emacs.d/elpa
+    rm -rf ~/.emacs.d/.cache
+    rm ~/.spacemacs
+
+    testdir=~/.emacs.d/tests/$test
+    if [ -f $testdir/dotspacemacs.el ]; then
+        cp $testdir/dotspacemacs.el ~/.spacemacs
+    fi
+    cd $testdir
+    make test || exit 2
+done