core: refactor layer system

TL;DR Should get 20~25% speed improvement on startup, should get a big
improvement when using ivy or helm SPC h SPC. Users with layers.el files
in their layers must use `configuration-layer/declare-used-layer`
instead of `configuration-layer/declare-layer`

The implementation of the layer system made heavy use of `object-assoc`
and `object-assoc-list` functions which are not efficient. This PR
mainly replaces those object lists with hash maps in order to index the
objects by their name and achieve an O(1) access time.

The old object lists `configuration-layer--layers` and
`configuration-layer--packages` have been each by two variables each:
- `configuration-layer--indexed-layers` which is a hash-map of all the
layer objects and `configuration-layer--used-layers` which is a list of
all _used_ layers symbols,
- symmetrically `configuration-layer--indexed-packages` which is a
hash-map of all the package objects and
`configuration-layer--used-packages` which is a list of all _used_
packages symbols.

The hash map `configuration-layer--layer-paths` is gone, now we create
directly layer objects when discovering the layers and set the :dir
property. Note that previously the layer paths were the parent directory
of the layer, now :dir is the layer path.

The function `configuration-layer//make-layer` is now similar to its
counterpart `configuration-layer//make-package` in the sense that it
takes an optional `obj` to be able to override its properties.

The functions `configuration-layer/declare-layer` and
`configuration-layer/declare-layers` now takes an optional parameter
`usedp` in order to declare used or not used layers. For convenience
new functions have been added: `configuration-layer/declare-used-layer`
and `configuration-layer/declare-used-layers`, users _must_ update all
occurrences of `configuration-layer/declare-layer` by
`configuration-layer/declare-used-layers` in their `layers.el` files.

`helm-spacemacs-help` and `ivy-spacemacs-help` are updated to match the
changes in `core-configuration-layer.el`.

Rename some variables to make them more explicit:
`configuration-layer-no-layer` -> `configuration-layer-exclude-all-layers`
`configuration-layer-distribution` -> `configuration-layer-force-distribution`

core/core-command-line.el

@@ -47,9 +47,9 @@ arguments is that we want to process these arguments as soon as possible."
           ("--insecure"
            (setq spacemacs-insecure t))
           ("--no-layer"
-           (setq configuration-layer-no-layer t))
+           (setq configuration-layer-exclude-all-layers t))
           ("--distribution"
-           (setq configuration-layer-distribution (intern (nth (1+ i) args))
+           (setq configuration-layer-force-distribution (intern (nth (1+ i) args))
                  i (1+ i)))
           ("--resume-layouts"
            (setq spacemacs-force-resume-layouts t))

core/core-dotspacemacs.el

@@ -656,7 +656,7 @@ error recovery."
   "Test settings in dotfile for correctness.
  Return non-nil if all the tests passed."
   (interactive)
-  (setq configuration-layer-paths (configuration-layer//discover-layers))
+  (configuration-layer/discover-layers)
   (let ((min-version "0.0"))
     ;; dotspacemacs-version not implemented yet
     ;; (if (version< dotspacemacs-version min-version)

core/core-spacemacs-buffer.el

@@ -508,7 +508,7 @@ border."
     (setq spacemacs-loading-value (1+ spacemacs-loading-value))
     (when (>= spacemacs-loading-counter spacemacs-loading-dots-chunk-threshold)
       (let ((suffix (format "> %s/%s" spacemacs-loading-value
-                            (length configuration-layer--packages))))
+                            (length configuration-layer--used-packages))))
         (setq spacemacs-loading-counter 0)
         (setq spacemacs-loading-string
               (make-string

layers/+completion/auto-completion/packages.el

@@ -189,7 +189,7 @@
       (define-key yas-minor-mode-map (kbd "M-s-/") 'yas-next-field)
       ;; configure snippet directories
       (let* ((spacemacs--auto-completion-dir
-              (configuration-layer/get-layer-property 'auto-completion :dir))
+              (configuration-layer/get-layer-local-dir 'auto-completion))
              (private-yas-dir (if auto-completion-private-snippets-directory
                                   auto-completion-private-snippets-directory
                                 (concat

layers/+completion/helm/local/helm-spacemacs-help/helm-spacemacs-help.el

@@ -33,11 +33,8 @@
 (require 'helm-org)
 (require 'core-configuration-layer)
 
-(defvar helm-spacemacs-help-all-layers nil
-  "Alist of all configuration layers.")
-
-(defvar helm-spacemacs-help-all-packages nil
-  "Hash table of all packages in all layers.")
+(defvar helm-spacemacs--initialized nil
+  "Non nil if helm-spacemacs is initialized.")
 
 ;;;###autoload
 (define-minor-mode helm-spacemacs-help-mode
@@ -46,14 +43,13 @@
   :global t)
 
 (defun helm-spacemacs-help//init (&optional arg)
-  (when (or arg (null helm-spacemacs-help-all-packages))
-    (mapc (lambda (layer) (push (configuration-layer/make-layer layer)
-                                helm-spacemacs-help-all-layers))
-          (configuration-layer/get-layers-list))
-    (let ((configuration-layer--inhibit-warnings t)
-          configuration-layer--packages)
-      (configuration-layer/get-packages helm-spacemacs-help-all-layers)
-      (setq helm-spacemacs-help-all-packages configuration-layer--packages))))
+  (when (or arg (null helm-spacemacs--initialized))
+    (let ((configuration-layer--load-packages-files t)
+          (configuration-layer--inhibit-warnings t))
+      (configuration-layer/discover-layers)
+      (configuration-layer/declare-layers (configuration-layer/get-layers-list))
+      (configuration-layer/make-all-packages)
+      (setq helm-spacemacs--initialized t))))
 
 ;;;###autoload
 (defun helm-spacemacs-help (arg)
@@ -224,8 +220,9 @@
 (defun helm-spacemacs-help//package-candidates ()
   "Return the sorted candidates for package source."
   (let (result)
-    (dolist (pkg helm-spacemacs-help-all-packages)
-      (let* ((owner (cfgl-package-get-safe-owner pkg))
+    (dolist (pkg-name (configuration-layer/get-packages-list))
+      (let* ((pkg (configuration-layer/get-package pkg-name))
+             (owner (cfgl-package-get-safe-owner pkg))
              ;; the notion of owner does not make sense if the layer is not used
              (init-type (if (configuration-layer/layer-usedp owner)
                             "owner" "init")))
@@ -295,15 +292,10 @@
   "Return the sorted candidates for all the dospacemacs variables."
   (sort (dotspacemacs/get-variable-string-list) 'string<))
 
-(defun helm-spacemacs-help//layer-action-open-file (file candidate &optional edit)
+(defun helm-spacemacs-help//layer-action-open-file
+    (file candidate &optional edit)
   "Open FILE of the passed CANDIDATE.  If EDIT is false, open in view mode."
-  (let ((path (if (and (equalp file "README.org") (equalp candidate "spacemacs"))
-                  ;; Readme for spacemacs is in the project root
-                  (ht-get configuration-layer-paths (intern candidate))
-                (file-name-as-directory
-                 (concat (ht-get configuration-layer-paths
-                                 (intern candidate))
-                         candidate)))))
+  (let ((path (configuration-layer/get-layer-path (intern candidate))))
     (if (and (equal (file-name-extension file) "org")
              (not helm-current-prefix-arg))
         (if edit
@@ -350,8 +342,7 @@
            (init-type (match-string 2 candidate))
            (layer (match-string 3 candidate))
            (path (file-name-as-directory
-                  (concat (ht-get configuration-layer-paths (intern layer))
-                          layer)))
+                  (configuration-layer/get-layer-path (intern layer))))
            (filename (concat path "packages.el")))
       (when (string-match-p "owner" init-type)
         (setq init-type "init"))

layers/+completion/ivy/local/ivy-spacemacs-help/ivy-spacemacs-help.el

@@ -31,20 +31,23 @@
 (require 'ivy)
 (require 'core-configuration-layer)
 
-(defvar ivy-spacemacs-help-all-layers nil
-  "Alist of all configuration layers.")
+(defvar ivy-spacemacs--initialized nil
+  "Non nil if ivy-spacemacs is initialized.")
 
-(defvar ivy-spacemacs-help-all-packages nil
-  "Hash table of all packages in all layers.")
+;; (defvar ivy-spacemacs-help-all-layers nil
+;;   "Alist of all configuration layers.")
+
+;; (defvar ivy-spacemacs-help-all-packages nil
+;;   "Hash table of all packages in all layers.")
 
 (defun ivy-spacemacs-help//init (&optional arg)
-  (when (or arg (null ivy-spacemacs-help-all-packages))
-    (mapc (lambda (layer) (push (configuration-layer/make-layer layer)
-                                ivy-spacemacs-help-all-layers))
-          (configuration-layer/get-layers-list))
-    (let (configuration-layer--packages)
-      (configuration-layer/get-packages ivy-spacemacs-help-all-layers)
-      (setq ivy-spacemacs-help-all-packages configuration-layer--packages))))
+  (when (or arg (null ivy-spacemacs--initialized))
+    (let ((configuration-layer--load-packages-files t)
+          (configuration-layer--inhibit-warnings t))
+      (configuration-layer/discover-layers)
+      (configuration-layer/declare-layers (configuration-layer/get-layers-list))
+      (configuration-layer/make-all-packages)
+      (setq ivy-spacemacs--initialized t))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Docs
@@ -124,24 +127,11 @@
 
 (defun ivy-spacemacs-help//layer-action-get-directory (candidate)
   "Get directory of layer passed CANDIDATE."
-  (let ((path (if (equalp candidate "spacemacs")
-                  ;; Readme for spacemacs is in the project root
-                  (ht-get configuration-layer-paths (intern candidate))
-                (file-name-as-directory
-                 (concat (ht-get configuration-layer-paths
-                                 (intern candidate))
-                         candidate)))))
-    path))
+  (configuration-layer/get-layer-path (intern candidate)))
 
 (defun ivy-spacemacs-help//layer-action-open-file (file candidate &optional edit)
   "Open FILE of the passed CANDIDATE.  If EDIT is false, open in view mode."
-  (let ((path (if (and (equalp file "README.org") (equalp candidate "spacemacs"))
-                  ;; Readme for spacemacs is in the project root
-                  (ht-get configuration-layer-paths (intern candidate))
-                (file-name-as-directory
-                 (concat (ht-get configuration-layer-paths
-                                 (intern candidate))
-                         candidate)))))
+  (let ((path (configuration-layer/get-layer-path (intern candidate))))
     (if (equal (file-name-extension file) "org")
         (if edit
             (find-file (concat path file))
@@ -213,35 +203,36 @@
         (left-column-width
          (number-to-string
           (cl-reduce
-           (lambda (a x) (max (length (symbol-name (oref x :name))) a))
-           ivy-spacemacs-help-all-layers :initial-value 0)))
+           (lambda (a x) (max (length (symbol-name x)) a))
+           (configuration-layer/get-layers-list) :initial-value 0)))
         (owners (cl-remove-duplicates
                  (mapcar (lambda (pkg)
-                           (car (oref pkg :owners)))
-                         ivy-spacemacs-help-all-packages))))
-    (dolist (pkg ivy-spacemacs-help-all-packages)
-      (push (list (format (concat "%-" left-column-width "S %s %s")
-                          (car (oref pkg :owners ))
-                          (propertize (symbol-name (oref pkg :name))
-                                      'face 'font-lock-type-face)
-                          (propertize
-                           (if (package-installed-p (oref pkg :name))
-                               "[installed]" "")
-                           'face 'font-lock-comment-face))
-                  (symbol-name
-                   (car (oref pkg :owners )))
-                  (symbol-name (oref pkg :name)))
-            result))
+                           (let ((obj (configuration-layer/get-package pkg)))
+                             (car (oref obj :owners))))
+                         (configuration-layer/get-packages-list)))))
+    (dolist (pkg-name (configuration-layer/get-packages-list))
+      (let ((pkg (configuration-layer/get-package pkg-name)))
+        (push (list (format (concat "%-" left-column-width "S %s %s")
+                            (car (oref pkg :owners ))
+                            (propertize (symbol-name (oref pkg :name))
+                                        'face 'font-lock-type-face)
+                            (propertize
+                             (if (package-installed-p (oref pkg :name))
+                                 "[installed]" "")
+                             'face 'font-lock-comment-face))
+                    (symbol-name
+                     (car (oref pkg :owners )))
+                    (symbol-name (oref pkg :name)))
+              result)))
     (dolist (layer (delq nil
                          (cl-remove-if
-                          (lambda (layer)
-                            (memq (oref layer :name) owners))
-                          ivy-spacemacs-help-all-layers)))
+                          (lambda (x) (memq x owners))
+                          (configuration-layer/get-layers-list))))
       (push (list (format (concat "%-" left-column-width "S %s")
-                          (oref layer :name)
+                          layer
                           (propertize "no packages"
                                       'face 'warning))
-                  (oref layer :name)
+                  layer
                   nil)
             result))
     (sort result (lambda (a b) (string< (car a) (car b))))))
@@ -253,9 +244,7 @@
     (let* ((layer-str (cadr args))
            (layer-sym (intern layer-str))
            (package-str (caddr args))
-           (path (file-name-as-directory
-                  (concat (ht-get configuration-layer-paths layer-sym)
-                          layer-str)))
+           (path (configuration-layer/get-layer-path layer-sym))
            (filename (concat path "packages.el")))
       (find-file filename)
       (goto-char (point-min))

layers/+distributions/spacemacs/layers.el

@@ -9,7 +9,7 @@
 ;;
 ;;; License: GPLv3
 
-(configuration-layer/declare-layers '(spacemacs-base
+(configuration-layer/declare-used-layers '(spacemacs-base
                                       spacemacs-completion
                                       spacemacs-layouts
                                       spacemacs-editing
@@ -25,4 +25,4 @@
 ;; want `helm' completion.
 (unless (or (configuration-layer/layer-usedp 'ivy)
             (configuration-layer/layer-usedp 'helm))
-  (configuration-layer/declare-layers '(helm)))
+  (configuration-layer/declare-used-layers '(helm)))

layers/+emacs/org/packages.el

@@ -114,7 +114,7 @@
 
       (with-eval-after-load 'org-indent
         (spacemacs|hide-lighter org-indent-mode))
-      (let ((dir (configuration-layer/get-layer-property 'org :dir)))
+      (let ((dir (configuration-layer/get-layer-local-dir 'org)))
         (setq org-export-async-init-file (concat dir "org-async-init.el")))
       (defmacro spacemacs|org-emphasize (fname char)
         "Make function for setting the emphasis in org mode"
@@ -137,7 +137,7 @@
           "a" 'org-edit-src-abort
           "k" 'org-edit-src-abort))
 
-      (let ((dir (configuration-layer/get-layer-property 'org :dir)))
+      (let ((dir (configuration-layer/get-layer-local-dir 'org)))
         (setq org-export-async-init-file (concat dir "org-async-init.el")))
       (defmacro spacemacs|org-emphasize (fname char)
         "Make function for setting the emphasis in org mode"

layers/+frameworks/django/layers.el

@@ -11,4 +11,4 @@
 
 ;; Prerequisites
 
-(configuration-layer/declare-layer 'python)
+(configuration-layer/declare-used-layer 'python)

layers/+frameworks/react/layers.el

@@ -9,4 +9,4 @@
 ;;
 ;;; License: GPLv3
 
-(configuration-layer/declare-layers '(html javascript))
+(configuration-layer/declare-used-layers '(html javascript))

layers/+frameworks/ruby-on-rails/layers.el

@@ -11,4 +11,4 @@
 
 ;; Prerequisites
 
-(configuration-layer/declare-layer 'ruby)
+(configuration-layer/declare-used-layer 'ruby)

layers/+tools/ansible/layers.el

@@ -9,4 +9,4 @@
 ;;
 ;;; License: GPLv3
 
-(configuration-layer/declare-layers '(yaml))
+(configuration-layer/declare-used-layers '(yaml))

layers/+tools/puppet/layers.el

@@ -9,4 +9,4 @@
 ;;
 ;;; License: GPLv3
 
-(configuration-layer/declare-layer 'ruby)
+(configuration-layer/declare-used-layer 'ruby)

layers/+tools/ycmd/packages.el

@@ -24,5 +24,5 @@
     :defer t
     :init
     (unless (boundp 'ycmd-global-config)
-      (let ((dir (configuration-layer/get-layer-property 'ycmd :dir)))
+      (let ((dir (configuration-layer/get-layer-local-dir 'ycmd)))
         (setq-default ycmd-global-config (concat dir "global_conf.py"))))))

tests/core/core-configuration-layer-ftest.el

@@ -11,23 +11,26 @@
 (require 'core-configuration-layer)
 
 ;; ---------------------------------------------------------------------------
-;; configuration-layer//declare-layers
+;; configuration-layer//declare-used-layers
 ;; ---------------------------------------------------------------------------
 
 (ert-deftest test-declare-layers--bootstrap-layer-always-first ()
   (let ((dotspacemacs-distribution 'spacemacs)
         (dotspacemacs-configuration-layers '(emacs-lisp
-                                             (git :variables foo 'bar))))
-    (let (configuration-layer--layers)
-      (configuration-layer//declare-layers)
-      (should (eq 'spacemacs-bootstrap
-                  (oref (first configuration-layer--layers) :name))))))
+                                             (git :variables foo 'bar)))
+        configuration-layer--used-layers
+        (configuration-layer--indexed-layers (make-hash-table :size 1024)))
+    (configuration-layer/discover-layers)
+    (configuration-layer//declare-used-layers)
+    (should (eq 'spacemacs-bootstrap
+                (first configuration-layer--used-layers)))))
 
 (ert-deftest test-declare-layers--distribution-layer-is-second ()
   (let ((dotspacemacs-distribution 'spacemacs-base)
         (dotspacemacs-configuration-layers '(emacs-lisp
-                                             (git :variables foo 'bar))))
-    (let (configuration-layer--layers)
-      (configuration-layer//declare-layers)
-      (should (eq 'spacemacs-base
-                  (oref (second configuration-layer--layers) :name))))))
+                                             (git :variables foo 'bar)))
+        configuration-layer--used-layers
+        (configuration-layer--indexed-layers (make-hash-table :size 1024)))
+    (configuration-layer/discover-layers)
+    (configuration-layer//declare-used-layers)
+    (should (eq 'spacemacs-base (second configuration-layer--used-layers)))))