feat: add token and lexer #7

	# This workflow will build a golang project
	# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go

	name: Go

	on:
	push:
	branches:
	- main
	pull_request:
	branches:
	- main

	jobs:
	build-test:
	runs-on: ubuntu-22.04
	steps:
	- uses: actions/checkout@v4

	- name: set up Go
	uses: actions/setup-go@v5
	with:
	go-version: '1.21'

	- name: install dependencies
	run: go get ./...

	- name: golangci-lint
	uses: golangci/golangci-lint-action@v4
	with:
	version: v1.54

	- name: build
	run: go build -race -v ./...

	- name: test
	run: \|
	go install github.com/onsi/ginkgo/v2/ginkgo
	ginkgo run -r -cover

Provide feedback