More efficient loading for default syntax files

This change introduces header files for syntax files. The header
files only contain the filetype and detection info and can be
parsed much faster than parsing a full yaml file. To determine
which filetype a file is, only scanning the headers is necessary
and afterwards only one yaml file needs to be parsed. Use the
make_headers.go file to generate the header files. Micro expects
that all default syntax files will have header files and that
custom user syntax files may or may not have them. Resolving
includes within syntax has not yet been implemented. This
optimization improves startup time.

Ref #1427

Makefile

@@ -1,15 +1,14 @@
 .PHONY: runtime
 
-VERSION := $(shell GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) \
+VERSION = $(shell GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) \
 	go run tools/build-version.go)
-HASH := $(shell git rev-parse --short HEAD)
+HASH = $(shell git rev-parse --short HEAD)
-DATE := $(shell GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) \
+DATE = $(shell GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) \
 	go run tools/build-date.go)
-ADDITIONAL_GO_LINKER_FLAGS := $(shell GOOS=$(shell go env GOHOSTOS) \
+ADDITIONAL_GO_LINKER_FLAGS = $(shell GOOS=$(shell go env GOHOSTOS) \
-	GOARCH=$(shell go env GOHOSTARCH) \
+	GOARCH=$(shell go env GOHOSTARCH))
-	go run tools/info-plist.go "$(VERSION)")
 GOBIN ?= $(shell go env GOPATH)/bin
-GOVARS := -X github.com/zyedidia/micro/internal/util.Version=$(VERSION) -X github.com/zyedidia/micro/internal/util.CommitHash=$(HASH) -X 'github.com/zyedidia/micro/internal/util.CompileDate=$(DATE)' -X github.com/zyedidia/micro/internal/util.Debug=OFF
+GOVARS = -X github.com/zyedidia/micro/internal/util.Version=$(VERSION) -X github.com/zyedidia/micro/internal/util.CommitHash=$(HASH) -X 'github.com/zyedidia/micro/internal/util.CompileDate=$(DATE)' -X github.com/zyedidia/micro/internal/util.Debug=OFF
 
 # Builds micro after checking dependencies but without updating the runtime
 build:
@@ -53,4 +52,3 @@ test:
 
 clean:
 	rm -f micro
-	rm -f runtime/syntax/*.hdr

internal/buffer/backup.go

@@ -36,7 +36,7 @@ func (b *Buffer) Backup(checkTime bool) error {
 
 	if checkTime {
 		sub := time.Now().Sub(b.lastbackup)
-		if sub < time.Duration(backup_time)*time.Millisecond {
+		if sub < time.Duration(backupTime)*time.Millisecond {
 			log.Println("Backup event but not enough time has passed", sub)
 			return nil
 		}

internal/buffer/buffer.go

@@ -25,7 +25,7 @@ import (
 	"golang.org/x/text/transform"
 )
 
-const backup_time = 8000
+const backupTime = 8000
 
 var (
 	OpenBuffers []*Buffer
@@ -113,7 +113,7 @@ type Buffer struct {
 	Messages []*Message
 
 	// counts the number of edits
-	// resets every backup_time edits
+	// resets every backupTime edits
 	lastbackup time.Time
 }
 
@@ -453,59 +453,91 @@ func (b *Buffer) UpdateRules() {
 	if !b.Type.Syntax {
 		return
 	}
-	rehighlight := false
+	syntaxFile := ""
-	var files []*highlight.File
+	ft := b.Settings["filetype"].(string)
-	for _, f := range config.ListRuntimeFiles(config.RTSyntax) {
+	var header *highlight.Header
+	for _, f := range config.ListRuntimeFiles(config.RTSyntaxHeader) {
 		data, err := f.Data()
 		if err != nil {
-			screen.TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
+			screen.TermMessage("Error loading syntax header file " + f.Name() + ": " + err.Error())
-		} else {
+			continue
-			file, err := highlight.ParseFile(data)
+		}
-			if err != nil {
-				screen.TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
-				continue
-			}
-			ftdetect, err := highlight.ParseFtDetect(file)
-			if err != nil {
-				screen.TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
-				continue
-			}
 
-			ft := b.Settings["filetype"].(string)
+		header, err = highlight.MakeHeader(data)
-			if (ft == "unknown" || ft == "") && !rehighlight {
+		if err != nil {
-				if highlight.MatchFiletype(ftdetect, b.Path, b.lines[0].data) {
+			screen.TermMessage("Error reading syntax header file", f.Name(), err)
-					header := new(highlight.Header)
+			continue
-					header.FileType = file.FileType
+		}
-					header.FtDetect = ftdetect
+
-					b.SyntaxDef, err = highlight.ParseDef(file, header)
+		if ft == "unknown" || ft == "" {
-					if err != nil {
+			if highlight.MatchFiletype(header.FtDetect, b.Path, b.lines[0].data) {
-						screen.TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
+				syntaxFile = f.Name()
-						continue
+				break
-					}
-					rehighlight = true
-				}
-			} else {
-				if file.FileType == ft && !rehighlight {
-					header := new(highlight.Header)
-					header.FileType = file.FileType
-					header.FtDetect = ftdetect
-					b.SyntaxDef, err = highlight.ParseDef(file, header)
-					if err != nil {
-						screen.TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
-						continue
-					}
-					rehighlight = true
-				}
 			}
-			files = append(files, file)
+		} else if header.FileType == ft {
+			syntaxFile = f.Name()
+			break
 		}
 	}
 
-	if b.SyntaxDef != nil {
+	if syntaxFile == "" {
-		highlight.ResolveIncludes(b.SyntaxDef, files)
+		// search for the syntax file in the user's custom syntax files
+		for _, f := range config.ListRealRuntimeFiles(config.RTSyntax) {
+			data, err := f.Data()
+			if err != nil {
+				screen.TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
+				continue
+			}
+
+			header, err = highlight.MakeHeaderYaml(data)
+			file, err := highlight.ParseFile(data)
+			if err != nil {
+				screen.TermMessage("Error parsing syntax file " + f.Name() + ": " + err.Error())
+				continue
+			}
+
+			if (ft == "unknown" || ft == "" && highlight.MatchFiletype(header.FtDetect, b.Path, b.lines[0].data)) || header.FileType == ft {
+				syndef, err := highlight.ParseDef(file, header)
+				if err != nil {
+					screen.TermMessage("Error parsing syntax file " + f.Name() + ": " + err.Error())
+					continue
+				}
+				b.SyntaxDef = syndef
+				break
+			}
+		}
+	} else {
+		for _, f := range config.ListRuntimeFiles(config.RTSyntax) {
+			if f.Name() == syntaxFile {
+				data, err := f.Data()
+				if err != nil {
+					screen.TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
+					continue
+				}
+
+				file, err := highlight.ParseFile(data)
+				if err != nil {
+					screen.TermMessage("Error parsing syntax file " + f.Name() + ": " + err.Error())
+					continue
+				}
+
+				syndef, err := highlight.ParseDef(file, header)
+				if err != nil {
+					screen.TermMessage("Error parsing syntax file " + f.Name() + ": " + err.Error())
+					continue
+				}
+				b.SyntaxDef = syndef
+				break
+			}
+		}
 	}
 
-	if b.Highlighter == nil || rehighlight {
+	// TODO: includes
+	// if b.SyntaxDef != nil {
+	// 	highlight.ResolveIncludes(b.SyntaxDef, files)
+	// }
+
+	if b.Highlighter == nil || syntaxFile != "" {
 		if b.SyntaxDef != nil {
 			b.Settings["filetype"] = b.SyntaxDef.FileType
 			b.Highlighter = highlight.NewHighlighter(b.SyntaxDef)

internal/config/rtfiles.go

@@ -32,6 +32,7 @@ type RuntimeFile interface {
 
 // allFiles contains all available files, mapped by filetype
 var allFiles [NumTypes][]RuntimeFile
+var realFiles [NumTypes][]RuntimeFile
 
 // some file on filesystem
 type realFile string
@@ -85,6 +86,12 @@ func AddRuntimeFile(fileType RTFiletype, file RuntimeFile) {
 	allFiles[fileType] = append(allFiles[fileType], file)
 }
 
+// AddRealRuntimeFile registers a file for the given filetype
+func AddRealRuntimeFile(fileType RTFiletype, file RuntimeFile) {
+	allFiles[fileType] = append(allFiles[fileType], file)
+	realFiles[fileType] = append(realFiles[fileType], file)
+}
+
 // AddRuntimeFilesFromDirectory registers each file from the given directory for
 // the filetype which matches the file-pattern
 func AddRuntimeFilesFromDirectory(fileType RTFiletype, directory, pattern string) {
@@ -92,7 +99,7 @@ func AddRuntimeFilesFromDirectory(fileType RTFiletype, directory, pattern string
 	for _, f := range files {
 		if ok, _ := filepath.Match(pattern, f.Name()); !f.IsDir() && ok {
 			fullPath := filepath.Join(directory, f.Name())
-			AddRuntimeFile(fileType, realFile(fullPath))
+			AddRealRuntimeFile(fileType, realFile(fullPath))
 		}
 	}
 }
@@ -127,6 +134,12 @@ func ListRuntimeFiles(fileType RTFiletype) []RuntimeFile {
 	return allFiles[fileType]
 }
 
+// ListRealRuntimeFiles lists all real runtime files (on disk) for a filetype
+// these runtime files will be ones defined by the user and loaded from the config directory
+func ListRealRuntimeFiles(fileType RTFiletype) []RuntimeFile {
+	return realFiles[fileType]
+}
+
 // InitRuntimeFiles initializes all assets file and the config directory
 func InitRuntimeFiles() {
 	add := func(fileType RTFiletype, dir, pattern string) {
@@ -136,7 +149,7 @@ func InitRuntimeFiles() {
 
 	add(RTColorscheme, "colorschemes", "*.micro")
 	add(RTSyntax, "syntax", "*.yaml")
-	add(RTSyntaxHeader, "header", "*.hdr")
+	add(RTSyntaxHeader, "syntax", "*.hdr")
 	add(RTHelp, "help", "*.md")
 
 	initlua := filepath.Join(ConfigDir, "init.lua")

pkg/highlight/parser.go

@@ -1,6 +1,7 @@
 package highlight
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"regexp"
@@ -41,6 +42,14 @@ type Header struct {
 	FtDetect [2]*regexp.Regexp
 }
 
+type HeaderYaml struct {
+	FileType string `yaml:"filetype"`
+	Detect   struct {
+		FNameRgx  string `yaml:"filename"`
+		HeaderRgx string `yaml:"header"`
+	} `yaml:"detect"`
+}
+
 type File struct {
 	FileType string
 
@@ -82,52 +91,67 @@ func init() {
 	Groups = make(map[string]Group)
 }
 
-func ParseFtDetect(file *File) (r [2]*regexp.Regexp, err error) {
+// MakeHeader takes a header (.hdr file) file and parses the header
-	defer func() {
+// Header files make parsing more efficient when you only want to compute
-		if r := recover(); r != nil {
+// on the headers of syntax files
-			var ok bool
+// A yaml file might take ~400us to parse while a header file only takes ~20us
-			err, ok = r.(error)
+func MakeHeader(data []byte) (*Header, error) {
-			if !ok {
+	lines := bytes.Split(data, []byte{'\n'})
-				err = fmt.Errorf("pkg: %v", r)
+	if len(lines) < 3 {
-			}
+		return nil, errors.New("Header file has incorrect format")
-		}
+	}
-	}()
+	header := new(Header)
+	var err error
+	header.FileType = string(lines[0])
+	fnameRgx := string(lines[1])
+	headerRgx := string(lines[2])
 
-	rules := file.yamlSrc
+	if fnameRgx == "" && headerRgx == "" {
-
+		return nil, errors.New("Syntax file must include at least one detection regex")
-	loaded := 0
-	for k, v := range rules {
-		if k == "detect" {
-			ftdetect := v.(map[interface{}]interface{})
-			if len(ftdetect) >= 1 {
-				syntax, err := regexp.Compile(ftdetect["filename"].(string))
-				if err != nil {
-					return r, err
-				}
-
-				r[0] = syntax
-			}
-			if len(ftdetect) >= 2 {
-				header, err := regexp.Compile(ftdetect["header"].(string))
-				if err != nil {
-					return r, err
-				}
-
-				r[1] = header
-			}
-			loaded++
-		}
-
-		if loaded >= 2 {
-			break
-		}
 	}
 
-	if loaded == 0 {
+	if fnameRgx != "" {
-		return r, errors.New("No detect regexes found")
+		header.FtDetect[0], err = regexp.Compile(fnameRgx)
+	}
+	if headerRgx != "" {
+		header.FtDetect[1], err = regexp.Compile(headerRgx)
 	}
 
-	return r, err
+	if err != nil {
+		return nil, err
+	}
+
+	return header, nil
+}
+
+// MakeHeaderYaml takes a yaml spec for a syntax file and parses the
+// header
+func MakeHeaderYaml(data []byte) (*Header, error) {
+	var hdrYaml HeaderYaml
+	err := yaml.Unmarshal(data, &hdrYaml)
+	if err != nil {
+		return nil, err
+	}
+
+	header := new(Header)
+	header.FileType = hdrYaml.FileType
+
+	if hdrYaml.Detect.FNameRgx == "" && hdrYaml.Detect.HeaderRgx == "" {
+		return nil, errors.New("Syntax file must include at least one detection regex")
+	}
+
+	if hdrYaml.Detect.FNameRgx != "" {
+		header.FtDetect[0], err = regexp.Compile(hdrYaml.Detect.FNameRgx)
+	}
+	if hdrYaml.Detect.HeaderRgx != "" {
+		header.FtDetect[1], err = regexp.Compile(hdrYaml.Detect.HeaderRgx)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return header, nil
 }
 
 func ParseFile(input []byte) (f *File, err error) {