zyedidia/micro
1
Fork 0
mirror of https://github.com/zyedidia/micro.git synced 2025-06-18 14:55:38 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

More efficient loading for default syntax files

Browse Source 
This change introduces header files for syntax files. The header
files only contain the filetype and detection info and can be
parsed much faster than parsing a full yaml file. To determine
which filetype a file is, only scanning the headers is necessary
and afterwards only one yaml file needs to be parsed. Use the
make_headers.go file to generate the header files. Micro expects
that all default syntax files will have header files and that
custom user syntax files may or may not have them. Resolving
includes within syntax has not yet been implemented. This
optimization improves startup time.

Ref #1427
This commit is contained in:
Zachary Yedidia 2019-12-28 21:26:22 -05:00
parent 8663014bbe
commit a61616d79e
6 changed files with 3065 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Makefile
View File

@ -1,15 +1,14 @@
.PHONY: runtime
VERSION := $(shell GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) \
VERSION = $(shell GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) \
go run tools/build-version.go)
HASH := $(shell git rev-parse --short HEAD)
DATE := $(shell GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) \
HASH = $(shell git rev-parse --short HEAD)
DATE = $(shell GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) \
go run tools/build-date.go)
ADDITIONAL_GO_LINKER_FLAGS := $(shell GOOS=$(shell go env GOHOSTOS) \
GOARCH=$(shell go env GOHOSTARCH) \
go run tools/info-plist.go "$(VERSION)")
ADDITIONAL_GO_LINKER_FLAGS = $(shell GOOS=$(shell go env GOHOSTOS) \
GOARCH=$(shell go env GOHOSTARCH))
GOBIN ?= $(shell go env GOPATH)/bin
GOVARS := -X github.com/zyedidia/micro/internal/util.Version=$(VERSION) -X github.com/zyedidia/micro/internal/util.CommitHash=$(HASH) -X 'github.com/zyedidia/micro/internal/util.CompileDate=$(DATE)' -X github.com/zyedidia/micro/internal/util.Debug=OFF
GOVARS = -X github.com/zyedidia/micro/internal/util.Version=$(VERSION) -X github.com/zyedidia/micro/internal/util.CommitHash=$(HASH) -X 'github.com/zyedidia/micro/internal/util.CompileDate=$(DATE)' -X github.com/zyedidia/micro/internal/util.Debug=OFF
# Builds micro after checking dependencies but without updating the runtime
build:
@ -53,4 +52,3 @@ test:
clean:
rm -f micro
rm -f runtime/syntax/*.hdr

2
internal/buffer/backup.go
View File

@ -36,7 +36,7 @@ func (b *Buffer) Backup(checkTime bool) error {
if checkTime {
sub := time.Now().Sub(b.lastbackup)
if sub < time.Duration(backup_time)*time.Millisecond {
if sub < time.Duration(backupTime)*time.Millisecond {
log.Println("Backup event but not enough time has passed", sub)
return nil
}

124
internal/buffer/buffer.go
View File

@ -25,7 +25,7 @@ import (
"golang.org/x/text/transform"
)
const backup_time = 8000
const backupTime = 8000
var (
OpenBuffers []*Buffer
@ -113,7 +113,7 @@ type Buffer struct {
Messages []*Message
// counts the number of edits
// resets every backup_time edits
// resets every backupTime edits
lastbackup time.Time
}
@ -453,59 +453,91 @@ func (b *Buffer) UpdateRules() {
if !b.Type.Syntax {
return
}
rehighlight := false
var files []*highlight.File
for _, f := range config.ListRuntimeFiles(config.RTSyntax) {
syntaxFile := ""
ft := b.Settings["filetype"].(string)
var header *highlight.Header
for _, f := range config.ListRuntimeFiles(config.RTSyntaxHeader) {
data, err := f.Data()
if err != nil {
screen.TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
} else {
file, err := highlight.ParseFile(data)
if err != nil {
screen.TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
continue
}
ftdetect, err := highlight.ParseFtDetect(file)
if err != nil {
screen.TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
continue
}
screen.TermMessage("Error loading syntax header file " + f.Name() + ": " + err.Error())
continue
}
ft := b.Settings["filetype"].(string)
if (ft == "unknown" || ft == "") && !rehighlight {
if highlight.MatchFiletype(ftdetect, b.Path, b.lines[0].data) {
header := new(highlight.Header)
header.FileType = file.FileType
header.FtDetect = ftdetect
b.SyntaxDef, err = highlight.ParseDef(file, header)
if err != nil {
screen.TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
continue
}
rehighlight = true
}
} else {
if file.FileType == ft && !rehighlight {
header := new(highlight.Header)
header.FileType = file.FileType
header.FtDetect = ftdetect
b.SyntaxDef, err = highlight.ParseDef(file, header)
if err != nil {
screen.TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
continue
}
rehighlight = true
}
header, err = highlight.MakeHeader(data)
if err != nil {
screen.TermMessage("Error reading syntax header file", f.Name(), err)
continue
}
if ft == "unknown" || ft == "" {
if highlight.MatchFiletype(header.FtDetect, b.Path, b.lines[0].data) {
syntaxFile = f.Name()
break
}
files = append(files, file)
} else if header.FileType == ft {
syntaxFile = f.Name()
break
}
}
if b.SyntaxDef != nil {
highlight.ResolveIncludes(b.SyntaxDef, files)
if syntaxFile == "" {
// search for the syntax file in the user's custom syntax files
for _, f := range config.ListRealRuntimeFiles(config.RTSyntax) {
data, err := f.Data()
if err != nil {
screen.TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
continue
}
header, err = highlight.MakeHeaderYaml(data)
file, err := highlight.ParseFile(data)
if err != nil {
screen.TermMessage("Error parsing syntax file " + f.Name() + ": " + err.Error())
continue
}
if (ft == "unknown" || ft == "" && highlight.MatchFiletype(header.FtDetect, b.Path, b.lines[0].data)) || header.FileType == ft {
syndef, err := highlight.ParseDef(file, header)
if err != nil {
screen.TermMessage("Error parsing syntax file " + f.Name() + ": " + err.Error())
continue
}
b.SyntaxDef = syndef
break
}
}
} else {
for _, f := range config.ListRuntimeFiles(config.RTSyntax) {
if f.Name() == syntaxFile {
data, err := f.Data()
if err != nil {
screen.TermMessage("Error loading syntax file " + f.Name() + ": " + err.Error())
continue
}
file, err := highlight.ParseFile(data)
if err != nil {
screen.TermMessage("Error parsing syntax file " + f.Name() + ": " + err.Error())
continue
}
syndef, err := highlight.ParseDef(file, header)
if err != nil {
screen.TermMessage("Error parsing syntax file " + f.Name() + ": " + err.Error())
continue
}
b.SyntaxDef = syndef
break
}
}
}
if b.Highlighter == nil || rehighlight {
// TODO: includes
// if b.SyntaxDef != nil {
// highlight.ResolveIncludes(b.SyntaxDef, files)
// }
if b.Highlighter == nil || syntaxFile != "" {
if b.SyntaxDef != nil {
b.Settings["filetype"] = b.SyntaxDef.FileType
b.Highlighter = highlight.NewHighlighter(b.SyntaxDef)

17
internal/config/rtfiles.go
View File

@ -32,6 +32,7 @@ type RuntimeFile interface {
// allFiles contains all available files, mapped by filetype
var allFiles [NumTypes][]RuntimeFile
var realFiles [NumTypes][]RuntimeFile
// some file on filesystem
type realFile string
@ -85,6 +86,12 @@ func AddRuntimeFile(fileType RTFiletype, file RuntimeFile) {
allFiles[fileType] = append(allFiles[fileType], file)
}
// AddRealRuntimeFile registers a file for the given filetype
func AddRealRuntimeFile(fileType RTFiletype, file RuntimeFile) {
allFiles[fileType] = append(allFiles[fileType], file)
realFiles[fileType] = append(realFiles[fileType], file)
}
// AddRuntimeFilesFromDirectory registers each file from the given directory for
// the filetype which matches the file-pattern
func AddRuntimeFilesFromDirectory(fileType RTFiletype, directory, pattern string) {
@ -92,7 +99,7 @@ func AddRuntimeFilesFromDirectory(fileType RTFiletype, directory, pattern string
for _, f := range files {
if ok, _ := filepath.Match(pattern, f.Name()); !f.IsDir() && ok {
fullPath := filepath.Join(directory, f.Name())
AddRuntimeFile(fileType, realFile(fullPath))
AddRealRuntimeFile(fileType, realFile(fullPath))
}
}
}
@ -127,6 +134,12 @@ func ListRuntimeFiles(fileType RTFiletype) []RuntimeFile {
return allFiles[fileType]
}
// ListRealRuntimeFiles lists all real runtime files (on disk) for a filetype
// these runtime files will be ones defined by the user and loaded from the config directory
func ListRealRuntimeFiles(fileType RTFiletype) []RuntimeFile {
return realFiles[fileType]
}
// InitRuntimeFiles initializes all assets file and the config directory
func InitRuntimeFiles() {
add := func(fileType RTFiletype, dir, pattern string) {
@ -136,7 +149,7 @@ func InitRuntimeFiles() {
add(RTColorscheme, "colorschemes", "*.micro")
add(RTSyntax, "syntax", "*.yaml")
add(RTSyntaxHeader, "header", "*.hdr")
add(RTSyntaxHeader, "syntax", "*.hdr")
add(RTHelp, "help", "*.md")
initlua := filepath.Join(ConfigDir, "init.lua")

2902
internal/config/runtime.go
View File

File diff suppressed because one or more lines are too long

106
pkg/highlight/parser.go
View File

@ -1,6 +1,7 @@
package highlight
import (
"bytes"
"errors"
"fmt"
"regexp"
@ -41,6 +42,14 @@ type Header struct {
FtDetect [2]*regexp.Regexp
}
type HeaderYaml struct {
FileType string `yaml:"filetype"`
Detect struct {
FNameRgx string `yaml:"filename"`
HeaderRgx string `yaml:"header"`
} `yaml:"detect"`
}
type File struct {
FileType string
@ -82,52 +91,67 @@ func init() {
Groups = make(map[string]Group)
}
func ParseFtDetect(file *File) (r [2]*regexp.Regexp, err error) {
defer func() {
if r := recover(); r != nil {
var ok bool
err, ok = r.(error)
if !ok {
err = fmt.Errorf("pkg: %v", r)
}
}
}()
// MakeHeader takes a header (.hdr file) file and parses the header
// Header files make parsing more efficient when you only want to compute
// on the headers of syntax files
// A yaml file might take ~400us to parse while a header file only takes ~20us
func MakeHeader(data []byte) (*Header, error) {
lines := bytes.Split(data, []byte{'\n'})
if len(lines) < 3 {
return nil, errors.New("Header file has incorrect format")
}
header := new(Header)
var err error
header.FileType = string(lines[0])
fnameRgx := string(lines[1])
headerRgx := string(lines[2])
rules := file.yamlSrc
loaded := 0
for k, v := range rules {
if k == "detect" {
ftdetect := v.(map[interface{}]interface{})
if len(ftdetect) >= 1 {
syntax, err := regexp.Compile(ftdetect["filename"].(string))
if err != nil {
return r, err
}
r[0] = syntax
}
if len(ftdetect) >= 2 {
header, err := regexp.Compile(ftdetect["header"].(string))
if err != nil {
return r, err
}
r[1] = header
}
loaded++
}
if loaded >= 2 {
break
}
if fnameRgx == "" && headerRgx == "" {
return nil, errors.New("Syntax file must include at least one detection regex")
}
if loaded == 0 {
return r, errors.New("No detect regexes found")
if fnameRgx != "" {
header.FtDetect[0], err = regexp.Compile(fnameRgx)
}
if headerRgx != "" {
header.FtDetect[1], err = regexp.Compile(headerRgx)
}
return r, err
if err != nil {
return nil, err
}
return header, nil
}
// MakeHeaderYaml takes a yaml spec for a syntax file and parses the
// header
func MakeHeaderYaml(data []byte) (*Header, error) {
var hdrYaml HeaderYaml
err := yaml.Unmarshal(data, &hdrYaml)
if err != nil {
return nil, err
}
header := new(Header)
header.FileType = hdrYaml.FileType
if hdrYaml.Detect.FNameRgx == "" && hdrYaml.Detect.HeaderRgx == "" {
return nil, errors.New("Syntax file must include at least one detection regex")
}
if hdrYaml.Detect.FNameRgx != "" {
header.FtDetect[0], err = regexp.Compile(hdrYaml.Detect.FNameRgx)
}
if hdrYaml.Detect.HeaderRgx != "" {
header.FtDetect[1], err = regexp.Compile(hdrYaml.Detect.HeaderRgx)
}
if err != nil {
return nil, err
}
return header, nil
}
func ParseFile(input []byte) (f *File, err error) {