commit 948e93ffac53c9607141b3c4093cabbcee84b2dd Author: Henrik Hautakoski Date: Tue Dec 10 19:51:21 2024 +0100 Initial Commit diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..f363cc5 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,39 @@ +root = true + +[*] +charset = utf-8 + +[*.{json,toml,yml,gyp}] +indent_style = space +indent_size = 2 + +[*.js] +indent_style = space +indent_size = 2 + +[*.{c,cc,h}] +indent_style = space +indent_size = 4 + +[*.rs] +indent_style = space +indent_size = 4 + +[*.{py,pyi}] +indent_style = space +indent_size = 4 + +[*.swift] +indent_style = space +indent_size = 4 + +[*.go] +indent_style = tab +indent_size = 8 + +[Makefile] +indent_style = tab +indent_size = 8 + +[parser.c] +indent_size = 2 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..4cb1058 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +* text=auto eol=lf + +src/*.json linguist-generated +src/parser.c linguist-generated +src/tree_sitter/* linguist-generated + +bindings/** linguist-generated +binding.gyp linguist-generated +setup.py linguist-generated +Makefile linguist-generated +Package.swift linguist-generated diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2fd9dac --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +# Rust artifacts +target/ + +# Node artifacts +build/ +prebuilds/ +node_modules/ +*.tgz + +# Swift artifacts +.build/ +Package.resolved + +# Go artifacts +_obj/ + +# Python artifacts +.venv/ +dist/ +*.egg-info +*.whl + +# C artifacts +*.a +*.so +*.so.* +*.dylib +*.dll +*.pc + +# Example dirs +/examples/*/ + +# Grammar volatiles +*.wasm +*.obj +*.o diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..af86707 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "tree-sitter-dotenv" +description = "Dotenv grammar for tree-sitter" +version = "0.0.1" +license = "MIT" +readme = "README.md" +keywords = ["incremental", "parsing", "tree-sitter", "dotenv"] +categories = ["parsing", "text-editors"] +repository = "https://github.com/tree-sitter/tree-sitter-dotenv" +edition = "2021" +autoexamples = false + +build = "bindings/rust/build.rs" +include = ["bindings/rust/*", "grammar.js", "queries/*", "src/*"] + +[lib] +path = "bindings/rust/lib.rs" + +[dependencies] +tree-sitter-language = "0.1" + +[build-dependencies] +cc = "1.0.87" + +[dev-dependencies] +tree-sitter = "0.23" diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fcc7915 --- /dev/null +++ b/Makefile @@ -0,0 +1,114 @@ +ifeq ($(OS),Windows_NT) +$(error Windows is not supported) +endif + +VERSION := 0.0.1 + +LANGUAGE_NAME := tree-sitter-dotenv + +# repository +SRC_DIR := src + +PARSER_REPO_URL := $(shell git -C $(SRC_DIR) remote get-url origin 2>/dev/null) + +ifeq ($(PARSER_URL),) + PARSER_URL := $(subst .git,,$(PARSER_REPO_URL)) +ifeq ($(shell echo $(PARSER_URL) | grep '^[a-z][-+.0-9a-z]*://'),) + PARSER_URL := $(subst :,/,$(PARSER_URL)) + PARSER_URL := $(subst git@,https://,$(PARSER_URL)) +endif +endif + +TS ?= tree-sitter + +# install directory layout +PREFIX ?= /usr/local +INCLUDEDIR ?= $(PREFIX)/include +LIBDIR ?= $(PREFIX)/lib +PCLIBDIR ?= $(LIBDIR)/pkgconfig + +# source/object files +PARSER := $(SRC_DIR)/parser.c +EXTRAS := $(filter-out $(PARSER),$(wildcard $(SRC_DIR)/*.c)) +OBJS := $(patsubst %.c,%.o,$(PARSER) $(EXTRAS)) + +# flags +ARFLAGS ?= rcs +override CFLAGS += -I$(SRC_DIR) -std=c11 -fPIC + +# ABI versioning +SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION))) +SONAME_MINOR := $(shell sed -n 's/#define LANGUAGE_VERSION //p' $(PARSER)) + +# OS-specific bits +ifeq ($(shell uname),Darwin) + SOEXT = dylib + SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT) + SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT) + LINKSHARED := $(LINKSHARED)-dynamiclib -Wl, + ifneq ($(ADDITIONAL_LIBS),) + LINKSHARED := $(LINKSHARED)$(ADDITIONAL_LIBS), + endif + LINKSHARED := $(LINKSHARED)-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SOEXTVER),-rpath,@executable_path/../Frameworks +else + SOEXT = so + SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR) + SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR) + LINKSHARED := $(LINKSHARED)-shared -Wl, + ifneq ($(ADDITIONAL_LIBS),) + LINKSHARED := $(LINKSHARED)$(ADDITIONAL_LIBS) + endif + LINKSHARED := $(LINKSHARED)-soname,lib$(LANGUAGE_NAME).$(SOEXTVER) +endif +ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),) + PCLIBDIR := $(PREFIX)/libdata/pkgconfig +endif + +all: lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) $(LANGUAGE_NAME).pc + +lib$(LANGUAGE_NAME).a: $(OBJS) + $(AR) $(ARFLAGS) $@ $^ + +lib$(LANGUAGE_NAME).$(SOEXT): $(OBJS) + $(CC) $(LDFLAGS) $(LINKSHARED) $^ $(LDLIBS) -o $@ +ifneq ($(STRIP),) + $(STRIP) $@ +endif + +$(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in + sed -e 's|@URL@|$(PARSER_URL)|' \ + -e 's|@VERSION@|$(VERSION)|' \ + -e 's|@LIBDIR@|$(LIBDIR)|' \ + -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \ + -e 's|@REQUIRES@|$(REQUIRES)|' \ + -e 's|@ADDITIONAL_LIBS@|$(ADDITIONAL_LIBS)|' \ + -e 's|=$(PREFIX)|=$${prefix}|' \ + -e 's|@PREFIX@|$(PREFIX)|' $< > $@ + +$(PARSER): $(SRC_DIR)/grammar.json + $(TS) generate --no-bindings $^ + +install: all + install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)' + install -m644 bindings/c/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h + install -m644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc + install -m644 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a + install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) + ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) + ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) + +uninstall: + $(RM) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a \ + '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) \ + '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) \ + '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) \ + '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h \ + '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc + +clean: + $(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) + +test: + $(TS) test + +.PHONY: all install uninstall clean test diff --git a/Package.swift b/Package.swift new file mode 100644 index 0000000..558017c --- /dev/null +++ b/Package.swift @@ -0,0 +1,37 @@ +// swift-tools-version:5.3 +import PackageDescription + +let package = Package( + name: "TreeSitterDotenv", + products: [ + .library(name: "TreeSitterDotenv", targets: ["TreeSitterDotenv"]), + ], + dependencies: [ + .package(url: "https://github.com/ChimeHQ/SwiftTreeSitter", from: "0.8.0"), + ], + targets: [ + .target( + name: "TreeSitterDotenv", + dependencies: [], + path: ".", + sources: [ + "src/parser.c", + // NOTE: if your language has an external scanner, add it here. + ], + resources: [ + .copy("queries") + ], + publicHeadersPath: "bindings/swift", + cSettings: [.headerSearchPath("src")] + ), + .testTarget( + name: "TreeSitterDotenvTests", + dependencies: [ + "SwiftTreeSitter", + "TreeSitterDotenv", + ], + path: "bindings/swift/TreeSitterDotenvTests" + ) + ], + cLanguageStandard: .c11 +) diff --git a/binding.gyp b/binding.gyp new file mode 100644 index 0000000..86675d9 --- /dev/null +++ b/binding.gyp @@ -0,0 +1,30 @@ +{ + "targets": [ + { + "target_name": "tree_sitter_dotenv_binding", + "dependencies": [ + " + +typedef struct TSLanguage TSLanguage; + +extern "C" TSLanguage *tree_sitter_dotenv(); + +// "tree-sitter", "language" hashed with BLAKE2 +const napi_type_tag LANGUAGE_TYPE_TAG = { + 0x8AF2E5212AD58ABF, 0xD5006CAD83ABBA16 +}; + +Napi::Object Init(Napi::Env env, Napi::Object exports) { + exports["name"] = Napi::String::New(env, "dotenv"); + auto language = Napi::External::New(env, tree_sitter_dotenv()); + language.TypeTag(&LANGUAGE_TYPE_TAG); + exports["language"] = language; + return exports; +} + +NODE_API_MODULE(tree_sitter_dotenv_binding, Init) diff --git a/bindings/node/binding_test.js b/bindings/node/binding_test.js new file mode 100644 index 0000000..afede30 --- /dev/null +++ b/bindings/node/binding_test.js @@ -0,0 +1,9 @@ +/// + +const assert = require("node:assert"); +const { test } = require("node:test"); + +test("can load grammar", () => { + const parser = new (require("tree-sitter"))(); + assert.doesNotThrow(() => parser.setLanguage(require("."))); +}); diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts new file mode 100644 index 0000000..efe259e --- /dev/null +++ b/bindings/node/index.d.ts @@ -0,0 +1,28 @@ +type BaseNode = { + type: string; + named: boolean; +}; + +type ChildNode = { + multiple: boolean; + required: boolean; + types: BaseNode[]; +}; + +type NodeInfo = + | (BaseNode & { + subtypes: BaseNode[]; + }) + | (BaseNode & { + fields: { [name: string]: ChildNode }; + children: ChildNode[]; + }); + +type Language = { + name: string; + language: unknown; + nodeTypeInfo: NodeInfo[]; +}; + +declare const language: Language; +export = language; diff --git a/bindings/node/index.js b/bindings/node/index.js new file mode 100644 index 0000000..6657bcf --- /dev/null +++ b/bindings/node/index.js @@ -0,0 +1,7 @@ +const root = require("path").join(__dirname, "..", ".."); + +module.exports = require("node-gyp-build")(root); + +try { + module.exports.nodeTypeInfo = require("../../src/node-types.json"); +} catch (_) {} diff --git a/bindings/python/tests/test_binding.py b/bindings/python/tests/test_binding.py new file mode 100644 index 0000000..c0b9c22 --- /dev/null +++ b/bindings/python/tests/test_binding.py @@ -0,0 +1,11 @@ +from unittest import TestCase + +import tree_sitter, tree_sitter_dotenv + + +class TestLanguage(TestCase): + def test_can_load_grammar(self): + try: + tree_sitter.Language(tree_sitter_dotenv.language()) + except Exception: + self.fail("Error loading Dotenv grammar") diff --git a/bindings/python/tree_sitter_dotenv/__init__.py b/bindings/python/tree_sitter_dotenv/__init__.py new file mode 100644 index 0000000..9603df9 --- /dev/null +++ b/bindings/python/tree_sitter_dotenv/__init__.py @@ -0,0 +1,42 @@ +"""Dotenv grammar for tree-sitter""" + +from importlib.resources import files as _files + +from ._binding import language + + +def _get_query(name, file): + query = _files(f"{__package__}.queries") / file + globals()[name] = query.read_text() + return globals()[name] + + +def __getattr__(name): + # NOTE: uncomment these to include any queries that this grammar contains: + + # if name == "HIGHLIGHTS_QUERY": + # return _get_query("HIGHLIGHTS_QUERY", "highlights.scm") + # if name == "INJECTIONS_QUERY": + # return _get_query("INJECTIONS_QUERY", "injections.scm") + # if name == "LOCALS_QUERY": + # return _get_query("LOCALS_QUERY", "locals.scm") + # if name == "TAGS_QUERY": + # return _get_query("TAGS_QUERY", "tags.scm") + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +__all__ = [ + "language", + # "HIGHLIGHTS_QUERY", + # "INJECTIONS_QUERY", + # "LOCALS_QUERY", + # "TAGS_QUERY", +] + + +def __dir__(): + return sorted(__all__ + [ + "__all__", "__builtins__", "__cached__", "__doc__", "__file__", + "__loader__", "__name__", "__package__", "__path__", "__spec__", + ]) diff --git a/bindings/python/tree_sitter_dotenv/__init__.pyi b/bindings/python/tree_sitter_dotenv/__init__.pyi new file mode 100644 index 0000000..abf6633 --- /dev/null +++ b/bindings/python/tree_sitter_dotenv/__init__.pyi @@ -0,0 +1,10 @@ +from typing import Final + +# NOTE: uncomment these to include any queries that this grammar contains: + +# HIGHLIGHTS_QUERY: Final[str] +# INJECTIONS_QUERY: Final[str] +# LOCALS_QUERY: Final[str] +# TAGS_QUERY: Final[str] + +def language() -> object: ... diff --git a/bindings/python/tree_sitter_dotenv/binding.c b/bindings/python/tree_sitter_dotenv/binding.c new file mode 100644 index 0000000..2c76a8b --- /dev/null +++ b/bindings/python/tree_sitter_dotenv/binding.c @@ -0,0 +1,27 @@ +#include + +typedef struct TSLanguage TSLanguage; + +TSLanguage *tree_sitter_dotenv(void); + +static PyObject* _binding_language(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) { + return PyCapsule_New(tree_sitter_dotenv(), "tree_sitter.Language", NULL); +} + +static PyMethodDef methods[] = { + {"language", _binding_language, METH_NOARGS, + "Get the tree-sitter language for this grammar."}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_binding", + .m_doc = NULL, + .m_size = -1, + .m_methods = methods +}; + +PyMODINIT_FUNC PyInit__binding(void) { + return PyModule_Create(&module); +} diff --git a/bindings/python/tree_sitter_dotenv/py.typed b/bindings/python/tree_sitter_dotenv/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/bindings/rust/build.rs b/bindings/rust/build.rs new file mode 100644 index 0000000..520fcc9 --- /dev/null +++ b/bindings/rust/build.rs @@ -0,0 +1,22 @@ +fn main() { + let src_dir = std::path::Path::new("src"); + + let mut c_config = cc::Build::new(); + c_config.std("c11").include(src_dir); + + #[cfg(target_env = "msvc")] + c_config.flag("-utf-8"); + + let parser_path = src_dir.join("parser.c"); + c_config.file(&parser_path); + println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap()); + + // NOTE: if your language uses an external scanner, uncomment this block: + /* + let scanner_path = src_dir.join("scanner.c"); + c_config.file(&scanner_path); + println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); + */ + + c_config.compile("tree-sitter-dotenv"); +} diff --git a/bindings/rust/lib.rs b/bindings/rust/lib.rs new file mode 100644 index 0000000..e3e6c58 --- /dev/null +++ b/bindings/rust/lib.rs @@ -0,0 +1,53 @@ +//! This crate provides Dotenv language support for the [tree-sitter][] parsing library. +//! +//! Typically, you will use the [LANGUAGE][] constant to add this language to a +//! tree-sitter [Parser][], and then use the parser to parse some code: +//! +//! ``` +//! let code = r#" +//! "#; +//! let mut parser = tree_sitter::Parser::new(); +//! let language = tree_sitter_dotenv::LANGUAGE; +//! parser +//! .set_language(&language.into()) +//! .expect("Error loading Dotenv parser"); +//! let tree = parser.parse(code, None).unwrap(); +//! assert!(!tree.root_node().has_error()); +//! ``` +//! +//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html +//! [tree-sitter]: https://tree-sitter.github.io/ + +use tree_sitter_language::LanguageFn; + +extern "C" { + fn tree_sitter_dotenv() -> *const (); +} + +/// The tree-sitter [`LanguageFn`][LanguageFn] for this grammar. +/// +/// [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html +pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_dotenv) }; + +/// The content of the [`node-types.json`][] file for this grammar. +/// +/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types +pub const NODE_TYPES: &str = include_str!("../../src/node-types.json"); + +// NOTE: uncomment these to include any queries that this grammar contains: + +// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm"); +// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm"); +// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm"); +// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm"); + +#[cfg(test)] +mod tests { + #[test] + fn test_can_load_grammar() { + let mut parser = tree_sitter::Parser::new(); + parser + .set_language(&super::LANGUAGE.into()) + .expect("Error loading Dotenv parser"); + } +} diff --git a/bindings/swift/TreeSitterDotenv/dotenv.h b/bindings/swift/TreeSitterDotenv/dotenv.h new file mode 100644 index 0000000..e05cbf3 --- /dev/null +++ b/bindings/swift/TreeSitterDotenv/dotenv.h @@ -0,0 +1,16 @@ +#ifndef TREE_SITTER_DOTENV_H_ +#define TREE_SITTER_DOTENV_H_ + +typedef struct TSLanguage TSLanguage; + +#ifdef __cplusplus +extern "C" { +#endif + +const TSLanguage *tree_sitter_dotenv(void); + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_DOTENV_H_ diff --git a/bindings/swift/TreeSitterDotenvTests/TreeSitterDotenvTests.swift b/bindings/swift/TreeSitterDotenvTests/TreeSitterDotenvTests.swift new file mode 100644 index 0000000..e9a9896 --- /dev/null +++ b/bindings/swift/TreeSitterDotenvTests/TreeSitterDotenvTests.swift @@ -0,0 +1,12 @@ +import XCTest +import SwiftTreeSitter +import TreeSitterDotenv + +final class TreeSitterDotenvTests: XCTestCase { + func testCanLoadGrammar() throws { + let parser = Parser() + let language = Language(language: tree_sitter_dotenv()) + XCTAssertNoThrow(try parser.setLanguage(language), + "Error loading Dotenv grammar") + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..697d9a8 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/pnx/tree-sitter-dotenv + +go 1.22 + +require github.com/tree-sitter/go-tree-sitter v1.23.1 diff --git a/grammar.js b/grammar.js new file mode 100644 index 0000000..cdf29e2 --- /dev/null +++ b/grammar.js @@ -0,0 +1,54 @@ +/// +// @ts-check + +module.exports = grammar({ + name: "dotenv", + + extras: $ => [ + /\s/ + ], + + externals: $ => [ + $._empty_value, + ], + + rules: { + document: $ => repeat(choice( + $.comment, + $.assignment, + )), + + assignment: $ => seq( + field("key", $.identifier), + "=", + field("value", $._value), + ), + + comment: _ => seq('#', /.*/), + + identifier: _ => token(/[A-Za-z_][A-Za-z0-9_]*/), + + _value: $ => choice( + $.string, + $.string_interpolation, + $.value, + alias($._empty_value, $.value), + ), + + string: $ => seq( + "'", + $.string_content, + "'", + ), + + string_interpolation: $ => seq( + '"', + alias($.string_interpolation_content, $.string_content), + '"', + ), + + string_content: _ => token(/[^']*/), + string_interpolation_content: _ => token(/[^"]*/), + value: _ => token(prec(-1, /[^\#\=\s]+/)), + }, +}); diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..4e089df --- /dev/null +++ b/package-lock.json @@ -0,0 +1,369 @@ +{ + "name": "tree-sitter-dotenv", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tree-sitter-dotenv", + "version": "0.0.1", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.0.0", + "node-gyp-build": "^4.8.1" + }, + "devDependencies": { + "prebuildify": "^6.0.1", + "tree-sitter-cli": "^0.23.2" + }, + "peerDependencies": { + "tree-sitter": "^0.21.1" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true, + "license": "ISC" + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true, + "license": "MIT" + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-abi": { + "version": "3.71.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.71.0.tgz", + "integrity": "sha512-SZ40vRiy/+wRTf21hxkkEjPJZpARzUMVcJoQse2EF8qkUWbbO2z7vd5oA/H6bVH6SZQ5STGcu0KRDS7biNRfxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-addon-api": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.3.0.tgz", + "integrity": "sha512-8VOpLHFrOQlAH+qA0ZzuGRlALRA6/LVh8QJldbrC4DY0hXoMP0l4Acq8TzFC018HztWiRqyCEj2aTWY2UvnJUg==", + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "license": "MIT", + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, + "node_modules/npm-run-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-3.1.0.tgz", + "integrity": "sha512-Dbl4A/VfiVGLgQv29URL9xshU8XDY1GeLy+fsaZ1AA8JDSfjvr5P5+pzRbWqRSBxk6/DW7MIh8lTM/PaGnP2kg==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/prebuildify": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/prebuildify/-/prebuildify-6.0.1.tgz", + "integrity": "sha512-8Y2oOOateom/s8dNBsGIcnm6AxPmLH4/nanQzL5lQMU+sC0CMhzARZHizwr36pUPLdvBnOkCNQzxg4djuFSgIw==", + "dev": true, + "license": "MIT", + "dependencies": { + "minimist": "^1.2.5", + "mkdirp-classic": "^0.5.3", + "node-abi": "^3.3.0", + "npm-run-path": "^3.1.0", + "pump": "^3.0.0", + "tar-fs": "^2.1.0" + }, + "bin": { + "prebuildify": "bin.js" + } + }, + "node_modules/pump": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "dev": true, + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dev": true, + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tree-sitter-cli": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.23.2.tgz", + "integrity": "sha512-kPPXprOqREX+C/FgUp2Qpt9jd0vSwn+hOgjzVv/7hapdoWpa+VeWId53rf4oNNd29ikheF12BYtGD/W90feMbA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "tree-sitter": "cli.js" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..e7886e1 --- /dev/null +++ b/package.json @@ -0,0 +1,52 @@ +{ + "name": "tree-sitter-dotenv", + "version": "0.0.1", + "description": "Dotenv grammar for tree-sitter", + "repository": "github:pnx/tree-sitter-dotenv", + "license": "MIT", + "main": "bindings/node", + "types": "bindings/node", + "keywords": [ + "incremental", + "parsing", + "tree-sitter", + "dotenv" + ], + "files": [ + "grammar.js", + "binding.gyp", + "prebuilds/**", + "bindings/node/*", + "queries/*", + "src/**", + "*.wasm" + ], + "dependencies": { + "node-addon-api": "^8.0.0", + "node-gyp-build": "^4.8.1" + }, + "devDependencies": { + "prebuildify": "^6.0.1", + "tree-sitter-cli": "^0.23.2" + }, + "peerDependencies": { + "tree-sitter": "^0.21.1" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + }, + "scripts": { + "install": "node-gyp-build", + "prestart": "tree-sitter build --wasm", + "start": "tree-sitter playground", + "test": "node --test bindings/node/*_test.js" + }, + "tree-sitter": [ + { + "scope": "source.dotenv", + "injection-regex": "^dotenv$" + } + ] +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..19cef34 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "tree-sitter-dotenv" +description = "Dotenv grammar for tree-sitter" +version = "0.0.1" +keywords = ["incremental", "parsing", "tree-sitter", "dotenv"] +classifiers = [ + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Topic :: Software Development :: Compilers", + "Topic :: Text Processing :: Linguistic", + "Typing :: Typed" +] +requires-python = ">=3.9" +license.text = "MIT" +readme = "README.md" + +[project.urls] +Homepage = "https://github.com/tree-sitter/tree-sitter-dotenv" + +[project.optional-dependencies] +core = ["tree-sitter~=0.22"] + +[tool.cibuildwheel] +build = "cp39-*" +build-frontend = "build" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..9f4f9f4 --- /dev/null +++ b/setup.py @@ -0,0 +1,62 @@ +from os.path import isdir, join +from platform import system + +from setuptools import Extension, find_packages, setup +from setuptools.command.build import build +from wheel.bdist_wheel import bdist_wheel + + +class Build(build): + def run(self): + if isdir("queries"): + dest = join(self.build_lib, "tree_sitter_dotenv", "queries") + self.copy_tree("queries", dest) + super().run() + + +class BdistWheel(bdist_wheel): + def get_tag(self): + python, abi, platform = super().get_tag() + if python.startswith("cp"): + python, abi = "cp39", "abi3" + return python, abi, platform + + +setup( + packages=find_packages("bindings/python"), + package_dir={"": "bindings/python"}, + package_data={ + "tree_sitter_dotenv": ["*.pyi", "py.typed"], + "tree_sitter_dotenv.queries": ["*.scm"], + }, + ext_package="tree_sitter_dotenv", + ext_modules=[ + Extension( + name="_binding", + sources=[ + "bindings/python/tree_sitter_dotenv/binding.c", + "src/parser.c", + # NOTE: if your language uses an external scanner, add it here. + ], + extra_compile_args=[ + "-std=c11", + "-fvisibility=hidden", + ] if system() != "Windows" else [ + "/std:c11", + "/utf-8", + ], + define_macros=[ + ("Py_LIMITED_API", "0x03090000"), + ("PY_SSIZE_T_CLEAN", None), + ("TREE_SITTER_HIDE_SYMBOLS", None), + ], + include_dirs=["src"], + py_limited_api=True, + ) + ], + cmdclass={ + "build": Build, + "bdist_wheel": BdistWheel + }, + zip_safe=False +) diff --git a/src/grammar.json b/src/grammar.json new file mode 100644 index 0000000..c93d50a --- /dev/null +++ b/src/grammar.json @@ -0,0 +1,172 @@ +{ + "name": "dotenv", + "rules": { + "document": { + "type": "REPEAT", + "content": { + "type": "CHOICE", + "members": [ + { + "type": "SYMBOL", + "name": "comment" + }, + { + "type": "SYMBOL", + "name": "assignment" + } + ] + } + }, + "assignment": { + "type": "SEQ", + "members": [ + { + "type": "FIELD", + "name": "key", + "content": { + "type": "SYMBOL", + "name": "identifier" + } + }, + { + "type": "STRING", + "value": "=" + }, + { + "type": "FIELD", + "name": "value", + "content": { + "type": "SYMBOL", + "name": "_value" + } + } + ] + }, + "comment": { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": "#" + }, + { + "type": "PATTERN", + "value": ".*" + } + ] + }, + "identifier": { + "type": "TOKEN", + "content": { + "type": "PATTERN", + "value": "[A-Za-z_][A-Za-z0-9_]*" + } + }, + "_value": { + "type": "CHOICE", + "members": [ + { + "type": "SYMBOL", + "name": "string" + }, + { + "type": "SYMBOL", + "name": "string_interpolation" + }, + { + "type": "SYMBOL", + "name": "value" + }, + { + "type": "ALIAS", + "content": { + "type": "SYMBOL", + "name": "_empty_value" + }, + "named": true, + "value": "value" + } + ] + }, + "string": { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": "'" + }, + { + "type": "SYMBOL", + "name": "string_content" + }, + { + "type": "STRING", + "value": "'" + } + ] + }, + "string_interpolation": { + "type": "SEQ", + "members": [ + { + "type": "STRING", + "value": "\"" + }, + { + "type": "ALIAS", + "content": { + "type": "SYMBOL", + "name": "string_interpolation_content" + }, + "named": true, + "value": "string_content" + }, + { + "type": "STRING", + "value": "\"" + } + ] + }, + "string_content": { + "type": "TOKEN", + "content": { + "type": "PATTERN", + "value": "[^']*" + } + }, + "string_interpolation_content": { + "type": "TOKEN", + "content": { + "type": "PATTERN", + "value": "[^\"]*" + } + }, + "value": { + "type": "TOKEN", + "content": { + "type": "PREC", + "value": -1, + "content": { + "type": "PATTERN", + "value": "[^\\#\\=\\s]+" + } + } + } + }, + "extras": [ + { + "type": "PATTERN", + "value": "\\s" + } + ], + "conflicts": [], + "precedences": [], + "externals": [ + { + "type": "SYMBOL", + "name": "_empty_value" + } + ], + "inline": [], + "supertypes": [] +} diff --git a/src/node-types.json b/src/node-types.json new file mode 100644 index 0000000..3d476e3 --- /dev/null +++ b/src/node-types.json @@ -0,0 +1,118 @@ +[ + { + "type": "assignment", + "named": true, + "fields": { + "key": { + "multiple": false, + "required": true, + "types": [ + { + "type": "identifier", + "named": true + } + ] + }, + "value": { + "multiple": false, + "required": true, + "types": [ + { + "type": "string", + "named": true + }, + { + "type": "string_interpolation", + "named": true + }, + { + "type": "value", + "named": true + } + ] + } + } + }, + { + "type": "comment", + "named": true, + "fields": {} + }, + { + "type": "document", + "named": true, + "fields": {}, + "children": { + "multiple": true, + "required": false, + "types": [ + { + "type": "assignment", + "named": true + }, + { + "type": "comment", + "named": true + } + ] + } + }, + { + "type": "string", + "named": true, + "fields": {}, + "children": { + "multiple": false, + "required": true, + "types": [ + { + "type": "string_content", + "named": true + } + ] + } + }, + { + "type": "string_interpolation", + "named": true, + "fields": {}, + "children": { + "multiple": false, + "required": true, + "types": [ + { + "type": "string_content", + "named": true + } + ] + } + }, + { + "type": "\"", + "named": false + }, + { + "type": "#", + "named": false + }, + { + "type": "'", + "named": false + }, + { + "type": "=", + "named": false + }, + { + "type": "identifier", + "named": true + }, + { + "type": "string_content", + "named": true + }, + { + "type": "value", + "named": true + } +] \ No newline at end of file diff --git a/src/parser.c b/src/parser.c new file mode 100644 index 0000000..62a3be9 --- /dev/null +++ b/src/parser.c @@ -0,0 +1,547 @@ +#include "tree_sitter/parser.h" + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" +#endif + +#define LANGUAGE_VERSION 14 +#define STATE_COUNT 16 +#define LARGE_STATE_COUNT 2 +#define SYMBOL_COUNT 18 +#define ALIAS_COUNT 0 +#define TOKEN_COUNT 11 +#define EXTERNAL_TOKEN_COUNT 1 +#define FIELD_COUNT 2 +#define MAX_ALIAS_SEQUENCE_LENGTH 3 +#define PRODUCTION_ID_COUNT 2 + +enum ts_symbol_identifiers { + anon_sym_EQ = 1, + anon_sym_POUND = 2, + aux_sym_comment_token1 = 3, + sym_identifier = 4, + anon_sym_SQUOTE = 5, + anon_sym_DQUOTE = 6, + sym_string_content = 7, + sym_string_interpolation_content = 8, + sym_value = 9, + sym__empty_value = 10, + sym_document = 11, + sym_assignment = 12, + sym_comment = 13, + sym__value = 14, + sym_string = 15, + sym_string_interpolation = 16, + aux_sym_document_repeat1 = 17, +}; + +static const char * const ts_symbol_names[] = { + [ts_builtin_sym_end] = "end", + [anon_sym_EQ] = "=", + [anon_sym_POUND] = "#", + [aux_sym_comment_token1] = "comment_token1", + [sym_identifier] = "identifier", + [anon_sym_SQUOTE] = "'", + [anon_sym_DQUOTE] = "\"", + [sym_string_content] = "string_content", + [sym_string_interpolation_content] = "string_content", + [sym_value] = "value", + [sym__empty_value] = "value", + [sym_document] = "document", + [sym_assignment] = "assignment", + [sym_comment] = "comment", + [sym__value] = "_value", + [sym_string] = "string", + [sym_string_interpolation] = "string_interpolation", + [aux_sym_document_repeat1] = "document_repeat1", +}; + +static const TSSymbol ts_symbol_map[] = { + [ts_builtin_sym_end] = ts_builtin_sym_end, + [anon_sym_EQ] = anon_sym_EQ, + [anon_sym_POUND] = anon_sym_POUND, + [aux_sym_comment_token1] = aux_sym_comment_token1, + [sym_identifier] = sym_identifier, + [anon_sym_SQUOTE] = anon_sym_SQUOTE, + [anon_sym_DQUOTE] = anon_sym_DQUOTE, + [sym_string_content] = sym_string_content, + [sym_string_interpolation_content] = sym_string_content, + [sym_value] = sym_value, + [sym__empty_value] = sym_value, + [sym_document] = sym_document, + [sym_assignment] = sym_assignment, + [sym_comment] = sym_comment, + [sym__value] = sym__value, + [sym_string] = sym_string, + [sym_string_interpolation] = sym_string_interpolation, + [aux_sym_document_repeat1] = aux_sym_document_repeat1, +}; + +static const TSSymbolMetadata ts_symbol_metadata[] = { + [ts_builtin_sym_end] = { + .visible = false, + .named = true, + }, + [anon_sym_EQ] = { + .visible = true, + .named = false, + }, + [anon_sym_POUND] = { + .visible = true, + .named = false, + }, + [aux_sym_comment_token1] = { + .visible = false, + .named = false, + }, + [sym_identifier] = { + .visible = true, + .named = true, + }, + [anon_sym_SQUOTE] = { + .visible = true, + .named = false, + }, + [anon_sym_DQUOTE] = { + .visible = true, + .named = false, + }, + [sym_string_content] = { + .visible = true, + .named = true, + }, + [sym_string_interpolation_content] = { + .visible = true, + .named = true, + }, + [sym_value] = { + .visible = true, + .named = true, + }, + [sym__empty_value] = { + .visible = true, + .named = true, + }, + [sym_document] = { + .visible = true, + .named = true, + }, + [sym_assignment] = { + .visible = true, + .named = true, + }, + [sym_comment] = { + .visible = true, + .named = true, + }, + [sym__value] = { + .visible = false, + .named = true, + }, + [sym_string] = { + .visible = true, + .named = true, + }, + [sym_string_interpolation] = { + .visible = true, + .named = true, + }, + [aux_sym_document_repeat1] = { + .visible = false, + .named = false, + }, +}; + +enum ts_field_identifiers { + field_key = 1, + field_value = 2, +}; + +static const char * const ts_field_names[] = { + [0] = NULL, + [field_key] = "key", + [field_value] = "value", +}; + +static const TSFieldMapSlice ts_field_map_slices[PRODUCTION_ID_COUNT] = { + [1] = {.index = 0, .length = 2}, +}; + +static const TSFieldMapEntry ts_field_map_entries[] = { + [0] = + {field_key, 0}, + {field_value, 2}, +}; + +static const TSSymbol ts_alias_sequences[PRODUCTION_ID_COUNT][MAX_ALIAS_SEQUENCE_LENGTH] = { + [0] = {0}, +}; + +static const uint16_t ts_non_terminal_alias_map[] = { + 0, +}; + +static const TSStateId ts_primary_state_ids[STATE_COUNT] = { + [0] = 0, + [1] = 1, + [2] = 2, + [3] = 3, + [4] = 4, + [5] = 5, + [6] = 6, + [7] = 7, + [8] = 8, + [9] = 9, + [10] = 10, + [11] = 11, + [12] = 12, + [13] = 13, + [14] = 14, + [15] = 15, +}; + +static bool ts_lex(TSLexer *lexer, TSStateId state) { + START_LEXER(); + eof = lexer->eof(lexer); + switch (state) { + case 0: + if (eof) ADVANCE(3); + if (lookahead == '"') ADVANCE(10); + if (lookahead == '#') ADVANCE(5); + if (lookahead == '\'') ADVANCE(9); + if (lookahead == '=') ADVANCE(4); + if (('\t' <= lookahead && lookahead <= '\r') || + lookahead == ' ') SKIP(0); + if (('A' <= lookahead && lookahead <= 'Z') || + lookahead == '_' || + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(8); + if (lookahead != 0) ADVANCE(15); + END_STATE(); + case 1: + if (lookahead == '"') ADVANCE(10); + if (lookahead == '\'') ADVANCE(9); + if (('\t' <= lookahead && lookahead <= '\r') || + lookahead == ' ') SKIP(1); + if (lookahead != 0 && + lookahead != '"' && + lookahead != '#' && + lookahead != '=') ADVANCE(15); + END_STATE(); + case 2: + if (eof) ADVANCE(3); + if (lookahead == '#') ADVANCE(5); + if (('\t' <= lookahead && lookahead <= '\r') || + lookahead == ' ') SKIP(2); + if (('A' <= lookahead && lookahead <= 'Z') || + lookahead == '_' || + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(8); + END_STATE(); + case 3: + ACCEPT_TOKEN(ts_builtin_sym_end); + END_STATE(); + case 4: + ACCEPT_TOKEN(anon_sym_EQ); + END_STATE(); + case 5: + ACCEPT_TOKEN(anon_sym_POUND); + END_STATE(); + case 6: + ACCEPT_TOKEN(aux_sym_comment_token1); + if (lookahead == '\t' || + (0x0b <= lookahead && lookahead <= '\r') || + lookahead == ' ') ADVANCE(6); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead)) ADVANCE(7); + END_STATE(); + case 7: + ACCEPT_TOKEN(aux_sym_comment_token1); + if (lookahead != 0 && + lookahead != '\n') ADVANCE(7); + END_STATE(); + case 8: + ACCEPT_TOKEN(sym_identifier); + if (('0' <= lookahead && lookahead <= '9') || + ('A' <= lookahead && lookahead <= 'Z') || + lookahead == '_' || + ('a' <= lookahead && lookahead <= 'z')) ADVANCE(8); + END_STATE(); + case 9: + ACCEPT_TOKEN(anon_sym_SQUOTE); + END_STATE(); + case 10: + ACCEPT_TOKEN(anon_sym_DQUOTE); + END_STATE(); + case 11: + ACCEPT_TOKEN(sym_string_content); + if (('\t' <= lookahead && lookahead <= '\r') || + lookahead == ' ') ADVANCE(11); + if (lookahead != 0 && + lookahead != '\'') ADVANCE(12); + END_STATE(); + case 12: + ACCEPT_TOKEN(sym_string_content); + if (lookahead != 0 && + lookahead != '\'') ADVANCE(12); + END_STATE(); + case 13: + ACCEPT_TOKEN(sym_string_interpolation_content); + if (('\t' <= lookahead && lookahead <= '\r') || + lookahead == ' ') ADVANCE(13); + if (lookahead != 0 && + lookahead != '"') ADVANCE(14); + END_STATE(); + case 14: + ACCEPT_TOKEN(sym_string_interpolation_content); + if (lookahead != 0 && + lookahead != '"') ADVANCE(14); + END_STATE(); + case 15: + ACCEPT_TOKEN(sym_value); + if (lookahead != 0 && + (lookahead < '\t' || '\r' < lookahead) && + lookahead != ' ' && + lookahead != '#' && + lookahead != '=') ADVANCE(15); + END_STATE(); + default: + return false; + } +} + +static const TSLexMode ts_lex_modes[STATE_COUNT] = { + [0] = {.lex_state = 0, .external_lex_state = 1}, + [1] = {.lex_state = 2}, + [2] = {.lex_state = 1, .external_lex_state = 1}, + [3] = {.lex_state = 2}, + [4] = {.lex_state = 2}, + [5] = {.lex_state = 2}, + [6] = {.lex_state = 2}, + [7] = {.lex_state = 2}, + [8] = {.lex_state = 2}, + [9] = {.lex_state = 6}, + [10] = {.lex_state = 0}, + [11] = {.lex_state = 0}, + [12] = {.lex_state = 11}, + [13] = {.lex_state = 13}, + [14] = {.lex_state = 0}, + [15] = {.lex_state = 0}, +}; + +static const uint16_t ts_parse_table[LARGE_STATE_COUNT][SYMBOL_COUNT] = { + [0] = { + [ts_builtin_sym_end] = ACTIONS(1), + [anon_sym_EQ] = ACTIONS(1), + [anon_sym_POUND] = ACTIONS(1), + [sym_identifier] = ACTIONS(1), + [anon_sym_SQUOTE] = ACTIONS(1), + [anon_sym_DQUOTE] = ACTIONS(1), + [sym_value] = ACTIONS(1), + [sym__empty_value] = ACTIONS(1), + }, + [1] = { + [sym_document] = STATE(11), + [sym_assignment] = STATE(3), + [sym_comment] = STATE(3), + [aux_sym_document_repeat1] = STATE(3), + [ts_builtin_sym_end] = ACTIONS(3), + [anon_sym_POUND] = ACTIONS(5), + [sym_identifier] = ACTIONS(7), + }, +}; + +static const uint16_t ts_small_parse_table[] = { + [0] = 5, + ACTIONS(9), 1, + anon_sym_SQUOTE, + ACTIONS(11), 1, + anon_sym_DQUOTE, + ACTIONS(13), 1, + sym_value, + ACTIONS(15), 1, + sym__empty_value, + STATE(6), 3, + sym__value, + sym_string, + sym_string_interpolation, + [18] = 4, + ACTIONS(5), 1, + anon_sym_POUND, + ACTIONS(7), 1, + sym_identifier, + ACTIONS(17), 1, + ts_builtin_sym_end, + STATE(4), 3, + sym_assignment, + sym_comment, + aux_sym_document_repeat1, + [33] = 4, + ACTIONS(19), 1, + ts_builtin_sym_end, + ACTIONS(21), 1, + anon_sym_POUND, + ACTIONS(24), 1, + sym_identifier, + STATE(4), 3, + sym_assignment, + sym_comment, + aux_sym_document_repeat1, + [48] = 1, + ACTIONS(27), 3, + ts_builtin_sym_end, + anon_sym_POUND, + sym_identifier, + [54] = 1, + ACTIONS(29), 3, + ts_builtin_sym_end, + anon_sym_POUND, + sym_identifier, + [60] = 1, + ACTIONS(31), 3, + ts_builtin_sym_end, + anon_sym_POUND, + sym_identifier, + [66] = 1, + ACTIONS(33), 3, + ts_builtin_sym_end, + anon_sym_POUND, + sym_identifier, + [72] = 1, + ACTIONS(35), 1, + aux_sym_comment_token1, + [76] = 1, + ACTIONS(37), 1, + anon_sym_EQ, + [80] = 1, + ACTIONS(39), 1, + ts_builtin_sym_end, + [84] = 1, + ACTIONS(41), 1, + sym_string_content, + [88] = 1, + ACTIONS(43), 1, + sym_string_interpolation_content, + [92] = 1, + ACTIONS(45), 1, + anon_sym_SQUOTE, + [96] = 1, + ACTIONS(47), 1, + anon_sym_DQUOTE, +}; + +static const uint32_t ts_small_parse_table_map[] = { + [SMALL_STATE(2)] = 0, + [SMALL_STATE(3)] = 18, + [SMALL_STATE(4)] = 33, + [SMALL_STATE(5)] = 48, + [SMALL_STATE(6)] = 54, + [SMALL_STATE(7)] = 60, + [SMALL_STATE(8)] = 66, + [SMALL_STATE(9)] = 72, + [SMALL_STATE(10)] = 76, + [SMALL_STATE(11)] = 80, + [SMALL_STATE(12)] = 84, + [SMALL_STATE(13)] = 88, + [SMALL_STATE(14)] = 92, + [SMALL_STATE(15)] = 96, +}; + +static const TSParseActionEntry ts_parse_actions[] = { + [0] = {.entry = {.count = 0, .reusable = false}}, + [1] = {.entry = {.count = 1, .reusable = false}}, RECOVER(), + [3] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_document, 0, 0, 0), + [5] = {.entry = {.count = 1, .reusable = true}}, SHIFT(9), + [7] = {.entry = {.count = 1, .reusable = true}}, SHIFT(10), + [9] = {.entry = {.count = 1, .reusable = true}}, SHIFT(12), + [11] = {.entry = {.count = 1, .reusable = true}}, SHIFT(13), + [13] = {.entry = {.count = 1, .reusable = false}}, SHIFT(6), + [15] = {.entry = {.count = 1, .reusable = true}}, SHIFT(6), + [17] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_document, 1, 0, 0), + [19] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_document_repeat1, 2, 0, 0), + [21] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_document_repeat1, 2, 0, 0), SHIFT_REPEAT(9), + [24] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_document_repeat1, 2, 0, 0), SHIFT_REPEAT(10), + [27] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_comment, 2, 0, 0), + [29] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_assignment, 3, 0, 1), + [31] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_string, 3, 0, 0), + [33] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_string_interpolation, 3, 0, 0), + [35] = {.entry = {.count = 1, .reusable = true}}, SHIFT(5), + [37] = {.entry = {.count = 1, .reusable = true}}, SHIFT(2), + [39] = {.entry = {.count = 1, .reusable = true}}, ACCEPT_INPUT(), + [41] = {.entry = {.count = 1, .reusable = true}}, SHIFT(14), + [43] = {.entry = {.count = 1, .reusable = true}}, SHIFT(15), + [45] = {.entry = {.count = 1, .reusable = true}}, SHIFT(7), + [47] = {.entry = {.count = 1, .reusable = true}}, SHIFT(8), +}; + +enum ts_external_scanner_symbol_identifiers { + ts_external_token__empty_value = 0, +}; + +static const TSSymbol ts_external_scanner_symbol_map[EXTERNAL_TOKEN_COUNT] = { + [ts_external_token__empty_value] = sym__empty_value, +}; + +static const bool ts_external_scanner_states[2][EXTERNAL_TOKEN_COUNT] = { + [1] = { + [ts_external_token__empty_value] = true, + }, +}; + +#ifdef __cplusplus +extern "C" { +#endif +void *tree_sitter_dotenv_external_scanner_create(void); +void tree_sitter_dotenv_external_scanner_destroy(void *); +bool tree_sitter_dotenv_external_scanner_scan(void *, TSLexer *, const bool *); +unsigned tree_sitter_dotenv_external_scanner_serialize(void *, char *); +void tree_sitter_dotenv_external_scanner_deserialize(void *, const char *, unsigned); + +#ifdef TREE_SITTER_HIDE_SYMBOLS +#define TS_PUBLIC +#elif defined(_WIN32) +#define TS_PUBLIC __declspec(dllexport) +#else +#define TS_PUBLIC __attribute__((visibility("default"))) +#endif + +TS_PUBLIC const TSLanguage *tree_sitter_dotenv(void) { + static const TSLanguage language = { + .version = LANGUAGE_VERSION, + .symbol_count = SYMBOL_COUNT, + .alias_count = ALIAS_COUNT, + .token_count = TOKEN_COUNT, + .external_token_count = EXTERNAL_TOKEN_COUNT, + .state_count = STATE_COUNT, + .large_state_count = LARGE_STATE_COUNT, + .production_id_count = PRODUCTION_ID_COUNT, + .field_count = FIELD_COUNT, + .max_alias_sequence_length = MAX_ALIAS_SEQUENCE_LENGTH, + .parse_table = &ts_parse_table[0][0], + .small_parse_table = ts_small_parse_table, + .small_parse_table_map = ts_small_parse_table_map, + .parse_actions = ts_parse_actions, + .symbol_names = ts_symbol_names, + .field_names = ts_field_names, + .field_map_slices = ts_field_map_slices, + .field_map_entries = ts_field_map_entries, + .symbol_metadata = ts_symbol_metadata, + .public_symbol_map = ts_symbol_map, + .alias_map = ts_non_terminal_alias_map, + .alias_sequences = &ts_alias_sequences[0][0], + .lex_modes = ts_lex_modes, + .lex_fn = ts_lex, + .external_scanner = { + &ts_external_scanner_states[0][0], + ts_external_scanner_symbol_map, + tree_sitter_dotenv_external_scanner_create, + tree_sitter_dotenv_external_scanner_destroy, + tree_sitter_dotenv_external_scanner_scan, + tree_sitter_dotenv_external_scanner_serialize, + tree_sitter_dotenv_external_scanner_deserialize, + }, + .primary_state_ids = ts_primary_state_ids, + }; + return &language; +} +#ifdef __cplusplus +} +#endif diff --git a/src/scanner.c b/src/scanner.c new file mode 100644 index 0000000..5a3764f --- /dev/null +++ b/src/scanner.c @@ -0,0 +1,45 @@ +#include "tree_sitter/parser.h" +#include "tree_sitter/alloc.h" +#include "tree_sitter/array.h" + +enum TokenType { + EMPTY_VALUE, +}; + +void *tree_sitter_dotenv_external_scanner_create(void) { + return NULL; +} + +void tree_sitter_dotenv_external_scanner_destroy(void *payload) {} + +unsigned tree_sitter_dotenv_external_scanner_serialize(void *payload, char *buffer) { + return 0; +} + +void tree_sitter_dotenv_external_scanner_deserialize(void *payload, const char *buffer, unsigned length) {} + +void advanceWS(TSLexer *lexer) { + while (lexer->lookahead == ' ' || lexer->lookahead == '\t') { + lexer->advance(lexer, true); + } +} + +bool tree_sitter_dotenv_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) { + + if (valid_symbols[EMPTY_VALUE]) { + advanceWS(lexer); + + if (lexer->lookahead == '\r') { + lexer->advance(lexer, true); + } + + if (lexer->eof(lexer) + || lexer->lookahead == '#' + || lexer->lookahead == '\n') { + lexer->result_symbol = EMPTY_VALUE; + return true; + } + } + + return false; +} diff --git a/src/tree_sitter/alloc.h b/src/tree_sitter/alloc.h new file mode 100644 index 0000000..1f4466d --- /dev/null +++ b/src/tree_sitter/alloc.h @@ -0,0 +1,54 @@ +#ifndef TREE_SITTER_ALLOC_H_ +#define TREE_SITTER_ALLOC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +// Allow clients to override allocation functions +#ifdef TREE_SITTER_REUSE_ALLOCATOR + +extern void *(*ts_current_malloc)(size_t); +extern void *(*ts_current_calloc)(size_t, size_t); +extern void *(*ts_current_realloc)(void *, size_t); +extern void (*ts_current_free)(void *); + +#ifndef ts_malloc +#define ts_malloc ts_current_malloc +#endif +#ifndef ts_calloc +#define ts_calloc ts_current_calloc +#endif +#ifndef ts_realloc +#define ts_realloc ts_current_realloc +#endif +#ifndef ts_free +#define ts_free ts_current_free +#endif + +#else + +#ifndef ts_malloc +#define ts_malloc malloc +#endif +#ifndef ts_calloc +#define ts_calloc calloc +#endif +#ifndef ts_realloc +#define ts_realloc realloc +#endif +#ifndef ts_free +#define ts_free free +#endif + +#endif + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_ALLOC_H_ diff --git a/src/tree_sitter/array.h b/src/tree_sitter/array.h new file mode 100644 index 0000000..15a3b23 --- /dev/null +++ b/src/tree_sitter/array.h @@ -0,0 +1,290 @@ +#ifndef TREE_SITTER_ARRAY_H_ +#define TREE_SITTER_ARRAY_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./alloc.h" + +#include +#include +#include +#include +#include + +#ifdef _MSC_VER +#pragma warning(disable : 4101) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" +#endif + +#define Array(T) \ + struct { \ + T *contents; \ + uint32_t size; \ + uint32_t capacity; \ + } + +/// Initialize an array. +#define array_init(self) \ + ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL) + +/// Create an empty array. +#define array_new() \ + { NULL, 0, 0 } + +/// Get a pointer to the element at a given `index` in the array. +#define array_get(self, _index) \ + (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index]) + +/// Get a pointer to the first element in the array. +#define array_front(self) array_get(self, 0) + +/// Get a pointer to the last element in the array. +#define array_back(self) array_get(self, (self)->size - 1) + +/// Clear the array, setting its size to zero. Note that this does not free any +/// memory allocated for the array's contents. +#define array_clear(self) ((self)->size = 0) + +/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is +/// less than the array's current capacity, this function has no effect. +#define array_reserve(self, new_capacity) \ + _array__reserve((Array *)(self), array_elem_size(self), new_capacity) + +/// Free any memory allocated for this array. Note that this does not free any +/// memory allocated for the array's contents. +#define array_delete(self) _array__delete((Array *)(self)) + +/// Push a new `element` onto the end of the array. +#define array_push(self, element) \ + (_array__grow((Array *)(self), 1, array_elem_size(self)), \ + (self)->contents[(self)->size++] = (element)) + +/// Increase the array's size by `count` elements. +/// New elements are zero-initialized. +#define array_grow_by(self, count) \ + do { \ + if ((count) == 0) break; \ + _array__grow((Array *)(self), count, array_elem_size(self)); \ + memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \ + (self)->size += (count); \ + } while (0) + +/// Append all elements from one array to the end of another. +#define array_push_all(self, other) \ + array_extend((self), (other)->size, (other)->contents) + +/// Append `count` elements to the end of the array, reading their values from the +/// `contents` pointer. +#define array_extend(self, count, contents) \ + _array__splice( \ + (Array *)(self), array_elem_size(self), (self)->size, \ + 0, count, contents \ + ) + +/// Remove `old_count` elements from the array starting at the given `index`. At +/// the same index, insert `new_count` new elements, reading their values from the +/// `new_contents` pointer. +#define array_splice(self, _index, old_count, new_count, new_contents) \ + _array__splice( \ + (Array *)(self), array_elem_size(self), _index, \ + old_count, new_count, new_contents \ + ) + +/// Insert one `element` into the array at the given `index`. +#define array_insert(self, _index, element) \ + _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element)) + +/// Remove one element from the array at the given `index`. +#define array_erase(self, _index) \ + _array__erase((Array *)(self), array_elem_size(self), _index) + +/// Pop the last element off the array, returning the element by value. +#define array_pop(self) ((self)->contents[--(self)->size]) + +/// Assign the contents of one array to another, reallocating if necessary. +#define array_assign(self, other) \ + _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self)) + +/// Swap one array with another +#define array_swap(self, other) \ + _array__swap((Array *)(self), (Array *)(other)) + +/// Get the size of the array contents +#define array_elem_size(self) (sizeof *(self)->contents) + +/// Search a sorted array for a given `needle` value, using the given `compare` +/// callback to determine the order. +/// +/// If an existing element is found to be equal to `needle`, then the `index` +/// out-parameter is set to the existing value's index, and the `exists` +/// out-parameter is set to true. Otherwise, `index` is set to an index where +/// `needle` should be inserted in order to preserve the sorting, and `exists` +/// is set to false. +#define array_search_sorted_with(self, compare, needle, _index, _exists) \ + _array__search_sorted(self, 0, compare, , needle, _index, _exists) + +/// Search a sorted array for a given `needle` value, using integer comparisons +/// of a given struct field (specified with a leading dot) to determine the order. +/// +/// See also `array_search_sorted_with`. +#define array_search_sorted_by(self, field, needle, _index, _exists) \ + _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists) + +/// Insert a given `value` into a sorted array, using the given `compare` +/// callback to determine the order. +#define array_insert_sorted_with(self, compare, value) \ + do { \ + unsigned _index, _exists; \ + array_search_sorted_with(self, compare, &(value), &_index, &_exists); \ + if (!_exists) array_insert(self, _index, value); \ + } while (0) + +/// Insert a given `value` into a sorted array, using integer comparisons of +/// a given struct field (specified with a leading dot) to determine the order. +/// +/// See also `array_search_sorted_by`. +#define array_insert_sorted_by(self, field, value) \ + do { \ + unsigned _index, _exists; \ + array_search_sorted_by(self, field, (value) field, &_index, &_exists); \ + if (!_exists) array_insert(self, _index, value); \ + } while (0) + +// Private + +typedef Array(void) Array; + +/// This is not what you're looking for, see `array_delete`. +static inline void _array__delete(Array *self) { + if (self->contents) { + ts_free(self->contents); + self->contents = NULL; + self->size = 0; + self->capacity = 0; + } +} + +/// This is not what you're looking for, see `array_erase`. +static inline void _array__erase(Array *self, size_t element_size, + uint32_t index) { + assert(index < self->size); + char *contents = (char *)self->contents; + memmove(contents + index * element_size, contents + (index + 1) * element_size, + (self->size - index - 1) * element_size); + self->size--; +} + +/// This is not what you're looking for, see `array_reserve`. +static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) { + if (new_capacity > self->capacity) { + if (self->contents) { + self->contents = ts_realloc(self->contents, new_capacity * element_size); + } else { + self->contents = ts_malloc(new_capacity * element_size); + } + self->capacity = new_capacity; + } +} + +/// This is not what you're looking for, see `array_assign`. +static inline void _array__assign(Array *self, const Array *other, size_t element_size) { + _array__reserve(self, element_size, other->size); + self->size = other->size; + memcpy(self->contents, other->contents, self->size * element_size); +} + +/// This is not what you're looking for, see `array_swap`. +static inline void _array__swap(Array *self, Array *other) { + Array swap = *other; + *other = *self; + *self = swap; +} + +/// This is not what you're looking for, see `array_push` or `array_grow_by`. +static inline void _array__grow(Array *self, uint32_t count, size_t element_size) { + uint32_t new_size = self->size + count; + if (new_size > self->capacity) { + uint32_t new_capacity = self->capacity * 2; + if (new_capacity < 8) new_capacity = 8; + if (new_capacity < new_size) new_capacity = new_size; + _array__reserve(self, element_size, new_capacity); + } +} + +/// This is not what you're looking for, see `array_splice`. +static inline void _array__splice(Array *self, size_t element_size, + uint32_t index, uint32_t old_count, + uint32_t new_count, const void *elements) { + uint32_t new_size = self->size + new_count - old_count; + uint32_t old_end = index + old_count; + uint32_t new_end = index + new_count; + assert(old_end <= self->size); + + _array__reserve(self, element_size, new_size); + + char *contents = (char *)self->contents; + if (self->size > old_end) { + memmove( + contents + new_end * element_size, + contents + old_end * element_size, + (self->size - old_end) * element_size + ); + } + if (new_count > 0) { + if (elements) { + memcpy( + (contents + index * element_size), + elements, + new_count * element_size + ); + } else { + memset( + (contents + index * element_size), + 0, + new_count * element_size + ); + } + } + self->size += new_count - old_count; +} + +/// A binary search routine, based on Rust's `std::slice::binary_search_by`. +/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`. +#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \ + do { \ + *(_index) = start; \ + *(_exists) = false; \ + uint32_t size = (self)->size - *(_index); \ + if (size == 0) break; \ + int comparison; \ + while (size > 1) { \ + uint32_t half_size = size / 2; \ + uint32_t mid_index = *(_index) + half_size; \ + comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \ + if (comparison <= 0) *(_index) = mid_index; \ + size -= half_size; \ + } \ + comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \ + if (comparison == 0) *(_exists) = true; \ + else if (comparison < 0) *(_index) += 1; \ + } while (0) + +/// Helper macro for the `_sorted_by` routines below. This takes the left (existing) +/// parameter by reference in order to work with the generic sorting function above. +#define _compare_int(a, b) ((int)*(a) - (int)(b)) + +#ifdef _MSC_VER +#pragma warning(default : 4101) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_ARRAY_H_ diff --git a/src/tree_sitter/parser.h b/src/tree_sitter/parser.h new file mode 100644 index 0000000..799f599 --- /dev/null +++ b/src/tree_sitter/parser.h @@ -0,0 +1,266 @@ +#ifndef TREE_SITTER_PARSER_H_ +#define TREE_SITTER_PARSER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#define ts_builtin_sym_error ((TSSymbol)-1) +#define ts_builtin_sym_end 0 +#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 + +#ifndef TREE_SITTER_API_H_ +typedef uint16_t TSStateId; +typedef uint16_t TSSymbol; +typedef uint16_t TSFieldId; +typedef struct TSLanguage TSLanguage; +#endif + +typedef struct { + TSFieldId field_id; + uint8_t child_index; + bool inherited; +} TSFieldMapEntry; + +typedef struct { + uint16_t index; + uint16_t length; +} TSFieldMapSlice; + +typedef struct { + bool visible; + bool named; + bool supertype; +} TSSymbolMetadata; + +typedef struct TSLexer TSLexer; + +struct TSLexer { + int32_t lookahead; + TSSymbol result_symbol; + void (*advance)(TSLexer *, bool); + void (*mark_end)(TSLexer *); + uint32_t (*get_column)(TSLexer *); + bool (*is_at_included_range_start)(const TSLexer *); + bool (*eof)(const TSLexer *); + void (*log)(const TSLexer *, const char *, ...); +}; + +typedef enum { + TSParseActionTypeShift, + TSParseActionTypeReduce, + TSParseActionTypeAccept, + TSParseActionTypeRecover, +} TSParseActionType; + +typedef union { + struct { + uint8_t type; + TSStateId state; + bool extra; + bool repetition; + } shift; + struct { + uint8_t type; + uint8_t child_count; + TSSymbol symbol; + int16_t dynamic_precedence; + uint16_t production_id; + } reduce; + uint8_t type; +} TSParseAction; + +typedef struct { + uint16_t lex_state; + uint16_t external_lex_state; +} TSLexMode; + +typedef union { + TSParseAction action; + struct { + uint8_t count; + bool reusable; + } entry; +} TSParseActionEntry; + +typedef struct { + int32_t start; + int32_t end; +} TSCharacterRange; + +struct TSLanguage { + uint32_t version; + uint32_t symbol_count; + uint32_t alias_count; + uint32_t token_count; + uint32_t external_token_count; + uint32_t state_count; + uint32_t large_state_count; + uint32_t production_id_count; + uint32_t field_count; + uint16_t max_alias_sequence_length; + const uint16_t *parse_table; + const uint16_t *small_parse_table; + const uint32_t *small_parse_table_map; + const TSParseActionEntry *parse_actions; + const char * const *symbol_names; + const char * const *field_names; + const TSFieldMapSlice *field_map_slices; + const TSFieldMapEntry *field_map_entries; + const TSSymbolMetadata *symbol_metadata; + const TSSymbol *public_symbol_map; + const uint16_t *alias_map; + const TSSymbol *alias_sequences; + const TSLexMode *lex_modes; + bool (*lex_fn)(TSLexer *, TSStateId); + bool (*keyword_lex_fn)(TSLexer *, TSStateId); + TSSymbol keyword_capture_token; + struct { + const bool *states; + const TSSymbol *symbol_map; + void *(*create)(void); + void (*destroy)(void *); + bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); + unsigned (*serialize)(void *, char *); + void (*deserialize)(void *, const char *, unsigned); + } external_scanner; + const TSStateId *primary_state_ids; +}; + +static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) { + uint32_t index = 0; + uint32_t size = len - index; + while (size > 1) { + uint32_t half_size = size / 2; + uint32_t mid_index = index + half_size; + TSCharacterRange *range = &ranges[mid_index]; + if (lookahead >= range->start && lookahead <= range->end) { + return true; + } else if (lookahead > range->end) { + index = mid_index; + } + size -= half_size; + } + TSCharacterRange *range = &ranges[index]; + return (lookahead >= range->start && lookahead <= range->end); +} + +/* + * Lexer Macros + */ + +#ifdef _MSC_VER +#define UNUSED __pragma(warning(suppress : 4101)) +#else +#define UNUSED __attribute__((unused)) +#endif + +#define START_LEXER() \ + bool result = false; \ + bool skip = false; \ + UNUSED \ + bool eof = false; \ + int32_t lookahead; \ + goto start; \ + next_state: \ + lexer->advance(lexer, skip); \ + start: \ + skip = false; \ + lookahead = lexer->lookahead; + +#define ADVANCE(state_value) \ + { \ + state = state_value; \ + goto next_state; \ + } + +#define ADVANCE_MAP(...) \ + { \ + static const uint16_t map[] = { __VA_ARGS__ }; \ + for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \ + if (map[i] == lookahead) { \ + state = map[i + 1]; \ + goto next_state; \ + } \ + } \ + } + +#define SKIP(state_value) \ + { \ + skip = true; \ + state = state_value; \ + goto next_state; \ + } + +#define ACCEPT_TOKEN(symbol_value) \ + result = true; \ + lexer->result_symbol = symbol_value; \ + lexer->mark_end(lexer); + +#define END_STATE() return result; + +/* + * Parse Table Macros + */ + +#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT) + +#define STATE(id) id + +#define ACTIONS(id) id + +#define SHIFT(state_value) \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .state = (state_value) \ + } \ + }} + +#define SHIFT_REPEAT(state_value) \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .state = (state_value), \ + .repetition = true \ + } \ + }} + +#define SHIFT_EXTRA() \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .extra = true \ + } \ + }} + +#define REDUCE(symbol_name, children, precedence, prod_id) \ + {{ \ + .reduce = { \ + .type = TSParseActionTypeReduce, \ + .symbol = symbol_name, \ + .child_count = children, \ + .dynamic_precedence = precedence, \ + .production_id = prod_id \ + }, \ + }} + +#define RECOVER() \ + {{ \ + .type = TSParseActionTypeRecover \ + }} + +#define ACCEPT_INPUT() \ + {{ \ + .type = TSParseActionTypeAccept \ + }} + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_PARSER_H_ diff --git a/test/corpus/assignments.txt b/test/corpus/assignments.txt new file mode 100644 index 0000000..8e6a453 --- /dev/null +++ b/test/corpus/assignments.txt @@ -0,0 +1,20 @@ +================================================================================ +Assignments +================================================================================ + +EMPTY= +EMPTY = +KEY_WITH_VALUE = value + +-------------------------------------------------------------------------------- + +(document + (assignment + key: (identifier) + value: (value)) + (assignment + key: (identifier) + value: (value)) + (assignment + key: (identifier) + value: (value))) diff --git a/test/corpus/comments.txt b/test/corpus/comments.txt new file mode 100644 index 0000000..dacd368 --- /dev/null +++ b/test/corpus/comments.txt @@ -0,0 +1,42 @@ +================================================================================ +Comments +================================================================================ + +# Comment + +# +# Multi line comment. +# + +EMPTY_WITH_COMMENT=# comment +EMPTY_WITH_COMMENT_WHITESPACE = # comment + +STRING_VALUE_WITH_COMMENT = 'string' # comment +STRING_VALUE_WITH_COMMENT = "string" # comment + +-------------------------------------------------------------------------------- + +(document + (comment) + (comment) + (comment) + (comment) + (assignment + key: (identifier) + value: (value)) + (comment) + (assignment + key: (identifier) + value: (value)) + (comment) + (assignment + key: (identifier) + value: (string + (string_content))) + (comment) + (assignment + key: (identifier) + value: (string_interpolation + (string_content))) + (comment)) + diff --git a/test/corpus/string_values.txt b/test/corpus/string_values.txt new file mode 100644 index 0000000..68845a2 --- /dev/null +++ b/test/corpus/string_values.txt @@ -0,0 +1,29 @@ +================================================================================ +String values +================================================================================ + +key = "quoted string" +key = "" + +key = 'single quoted string' +key = '' + +-------------------------------------------------------------------------------- + +(document + (assignment + (identifier) + (string_interpolation + (string_content))) + (assignment + (identifier) + (string_interpolation + (string_content))) + (assignment + (identifier) + (string + (string_content))) + (assignment + (identifier) + (string + (string_content))))