diff --git a/.github/workflows/check-cli-documentation.yml b/.github/workflows/check-cli-documentation.yml deleted file mode 100644 index be50351518..0000000000 --- a/.github/workflows/check-cli-documentation.yml +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2024 Democratized Data Foundation -# -# Use of this software is governed by the Business Source License -# included in the file licenses/BSL.txt. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0, included in the file -# licenses/APL.txt. - -# This workflow checks that all CLI documentation is up to date. -# If the documentation is not up to date then this action will fail. -name: Check CLI Documentation Workflow - -on: - pull_request: - branches: - - master - - develop - - push: - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - branches: - - master - - develop - -jobs: - check-cli-documentation: - name: Check cli documentation job - - runs-on: ubuntu-latest - - steps: - - name: Checkout code into the directory - uses: actions/checkout@v3 - - - name: Setup Go environment explicitly - uses: actions/setup-go@v3 - with: - go-version: "1.21" - check-latest: true - - - name: Try generating cli documentation - run: make docs:cli - - - name: Check no new changes exist - uses: tj-actions/verify-changed-files@v20 - with: - fail-if-changed: true - files: | - docs/website/references/cli diff --git a/.github/workflows/check-documentation.yml b/.github/workflows/check-documentation.yml new file mode 100644 index 0000000000..97214d515b --- /dev/null +++ b/.github/workflows/check-documentation.yml @@ -0,0 +1,102 @@ +# Copyright 2024 Democratized Data Foundation +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +# This workflow checks that all documentation is up to date. +# If any documentation is not up to date then this action will fail. +name: Check Documentation Workflow + +on: + pull_request: + branches: + - master + - develop + + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + branches: + - master + - develop + +jobs: + check-cli-documentation: + name: Check cli documentation job + + runs-on: ubuntu-latest + + steps: + - name: Checkout code into the directory + uses: actions/checkout@v3 + + - name: Setup Go environment explicitly + uses: actions/setup-go@v3 + with: + go-version: "1.21" + check-latest: true + + - name: Try generating cli documentation + run: make docs:cli + + - name: Check no new changes exist + uses: tj-actions/verify-changed-files@v20 + with: + fail-if-changed: true + files: | + docs/website/references/cli + + check-http-documentation: + name: Check http documentation job + + runs-on: ubuntu-latest + + steps: + - name: Checkout code into the directory + uses: actions/checkout@v3 + + - name: Setup Go environment explicitly + uses: actions/setup-go@v3 + with: + go-version: "1.21" + check-latest: true + + - name: Try generating http documentation + run: make docs:http + + - name: Check no new changes exist + uses: tj-actions/verify-changed-files@v20 + with: + fail-if-changed: true + files: | + docs/website/references/http + + check-readme-toc: + name: Check readme toc job + + runs-on: ubuntu-latest + + steps: + - name: Checkout code into the directory + uses: actions/checkout@v3 + + - name: Setup Go environment explicitly + uses: actions/setup-go@v3 + with: + go-version: "1.21" + check-latest: true + + - name: Try generating readme toc + run: make toc + + - name: Check no new changes exist + uses: tj-actions/verify-changed-files@v20 + with: + fail-if-changed: true + files: | + README.md diff --git a/.github/workflows/check-http-documentation.yml b/.github/workflows/check-http-documentation.yml deleted file mode 100644 index e68471c162..0000000000 --- a/.github/workflows/check-http-documentation.yml +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2024 Democratized Data Foundation -# -# Use of this software is governed by the Business Source License -# included in the file licenses/BSL.txt. -# -# As of the Change Date specified in that file, in accordance with -# the Business Source License, use of this software will be governed -# by the Apache License, Version 2.0, included in the file -# licenses/APL.txt. - -# This workflow checks that all HTTP documentation is up to date. -# If the documentation is not up to date then this action will fail. -name: Check HTTP Documentation Workflow - -on: - pull_request: - branches: - - master - - develop - - push: - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - branches: - - master - - develop - -jobs: - check-http-documentation: - name: Check http documentation job - - runs-on: ubuntu-latest - - steps: - - name: Checkout code into the directory - uses: actions/checkout@v3 - - - name: Setup Go environment explicitly - uses: actions/setup-go@v3 - with: - go-version: "1.21" - check-latest: true - - - name: Try generating http documentation - run: make docs:http - - - name: Check no new changes exist - uses: tj-actions/verify-changed-files@v20 - with: - fail-if-changed: true - files: | - docs/website/references/http diff --git a/Makefile b/Makefile index c4c02da3d7..f4cfa7a7bd 100644 --- a/Makefile +++ b/Makefile @@ -357,6 +357,7 @@ docs: @$(MAKE) docs\:cli @$(MAKE) docs\:manpages @$(MAKE) docs\:http + @$(MAKE) toc .PHONY: docs\:cli docs\:cli: @@ -375,3 +376,7 @@ docs\:manpages: docs\:godoc: godoc -http=:6060 # open http://localhost:6060/pkg/github.com/sourcenetwork/defradb/ + +.PHONY: toc +toc: + bash tools/scripts/md-toc/gh-md-toc --insert --no-backup --hide-footer --skip-header README.md diff --git a/README.md b/README.md index 71e3ff856c..db0a3598b7 100644 --- a/README.md +++ b/README.md @@ -15,29 +15,29 @@ DefraDB is a user-centric database that prioritizes data ownership, personal pri Read the documentation on [docs.source.network](https://docs.source.network/). + ## Table of Contents -- [Install](#install) -- [Key Management](#key-management) -- [Start](#start) -- [Configuration](#configuration) -- [External port binding](#external-port-binding) -- [Add a schema type](#add-a-schema-type) -- [Create a document instance](#create-a-document-instance) -- [Query documents](#query-documents) -- [Obtain document commits](#obtain-document-commits) -- [DefraDB Query Language (DQL)](#defradb-query-language-dql) -- [Peer-to-peer data synchronization](#peer-to-peer-data-synchronization) - - [Pubsub example](#pubsub-example) - - [Collection subscription example](#collection-subscription-example) - - [Replicator example](#replicator-example) -- [Securing the HTTP API with TLS](#securing-the-http-api-with-tls) -- [Access Control System](#access-control-system) -- [Supporting CORS](#supporting-cors) -- [Backing up and restoring](#backing-up-and-restoring) -- [Community](#community) -- [Licensing](#licensing) -- [Contributors](#contributors) + + * [Install](#install) + * [Key Management](#key-management) + * [Start](#start) + * [Configuration](#configuration) + * [External port binding](#external-port-binding) + * [Add a schema type](#add-a-schema-type) + * [Create a document](#create-a-document) + * [Query documents](#query-documents) + * [Obtain document commits](#obtain-document-commits) + * [DefraDB Query Language (DQL)](#defradb-query-language-dql) + * [Peer-to-peer data synchronization](#peer-to-peer-data-synchronization) + * [Securing the HTTP API with TLS](#securing-the-http-api-with-tls) + * [Access Control System](#access-control-system) + * [Supporting CORS](#supporting-cors) + * [Backing up and restoring](#backing-up-and-restoring) + * [Community](#community) + * [Licensing](#licensing) + * [Contributors](#contributors) + DISCLAIMER: At this early stage, DefraDB does not offer data encryption, and the default configuration exposes the database to the network. The software is provided "as is" and is not guaranteed to be stable, secure, or error-free. We encourage you to experiment with DefraDB and provide feedback, but please do not use it for production purposes until it has been thoroughly tested and developed. diff --git a/tools/scripts/md-toc/README.md b/tools/scripts/md-toc/README.md new file mode 100644 index 0000000000..bad07bc32f --- /dev/null +++ b/tools/scripts/md-toc/README.md @@ -0,0 +1,443 @@ +gh-md-toc +========= + +[![CI](https://github.com/ekalinin/github-markdown-toc/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/ekalinin/github-markdown-toc/actions/workflows/ci.yml) +![GitHub release (latest by date)](https://img.shields.io/github/v/release/ekalinin/github-markdown-toc) + +gh-md-toc — is for you if you **want to generate TOC** (Table Of Content) for a README.md or +a GitHub wiki page **without installing additional software**. + +It's my try to fix a problem: + + * [github/issues/215](https://github.com/isaacs/github/issues/215) + +gh-md-toc is able to process: + + * stdin + * local files (markdown files in local file system) + * remote files (html files on github.com) + +gh-md-toc tested on Ubuntu, and macOS High Sierra (gh-md-toc release 0.4.9). If you want it on Windows, you +better to use a golang based implementation: + + * [github-markdown-toc.go](https://github.com/ekalinin/github-markdown-toc.go) + +It's more solid, reliable and with ability of a parallel processing. And +absolutely without dependencies. + +Table of contents +================= + + + * [Installation](#installation) + * [Usage](#usage) + * [STDIN](#stdin) + * [Local files](#local-files) + * [Remote files](#remote-files) + * [Multiple files](#multiple-files) + * [Combo](#combo) + * [Auto insert and update TOC](#auto-insert-and-update-toc) + * [GitHub token](#github-token) + * [TOC generation with Github Actions](#toc-generation-with-github-actions) + * [Tests](#tests) + * [Dependency](#dependency) + * [Docker](#docker) + * [Local](#local) + * [Public](#public) + + + +Installation +============ + +Linux (manual installation) +```bash +$ wget https://raw.githubusercontent.com/ekalinin/github-markdown-toc/master/gh-md-toc +$ chmod a+x gh-md-toc +``` + +MacOS (manual installation) +```bash +$ curl https://raw.githubusercontent.com/ekalinin/github-markdown-toc/master/gh-md-toc -o gh-md-toc +$ chmod a+x gh-md-toc +``` + +Linux or MacOS (using [Basher](https://github.com/basherpm/basher)) +```bash +$ basher install ekalinin/github-markdown-toc +# `gh-md-toc` will automatically be available in the PATH +``` + +Usage +===== + + +STDIN +----- + +Here's an example of TOC creating for markdown from STDIN: + +```bash +➥ cat ~/projects/Dockerfile.vim/README.md | ./gh-md-toc - + * [Dockerfile.vim](#dockerfilevim) + * [Screenshot](#screenshot) + * [Installation](#installation) + * [OR using Pathogen:](#or-using-pathogen) + * [OR using Vundle:](#or-using-vundle) + * [License](#license) +``` + +Local files +----------- + +Here's an example of TOC creating for a local README.md: + +```bash +➥ ./gh-md-toc ~/projects/Dockerfile.vim/README.md + + +Table of Contents +================= + + * [Dockerfile.vim](#dockerfilevim) + * [Screenshot](#screenshot) + * [Installation](#installation) + * [OR using Pathogen:](#or-using-pathogen) + * [OR using Vundle:](#or-using-vundle) + * [License](#license) +``` + +Remote files +------------ + +And here's an example, when you have a README.md like this: + + * [README.md without TOC](https://github.com/ekalinin/envirius/blob/f939d3b6882bfb6ecb28ef7b6e62862f934ba945/README.md) + +And you want to generate TOC for it. + +There is nothing easier: + +```bash +➥ ./gh-md-toc https://github.com/ekalinin/envirius/blob/master/README.md + +Table of Contents +================= + + * [envirius](#envirius) + * [Idea](#idea) + * [Features](#features) + * [Installation](#installation) + * [Uninstallation](#uninstallation) + * [Available plugins](#available-plugins) + * [Usage](#usage) + * [Check available plugins](#check-available-plugins) + * [Check available versions for each plugin](#check-available-versions-for-each-plugin) + * [Create an environment](#create-an-environment) + * [Activate/deactivate environment](#activatedeactivate-environment) + * [Activating in a new shell](#activating-in-a-new-shell) + * [Activating in the same shell](#activating-in-the-same-shell) + * [Get list of environments](#get-list-of-environments) + * [Get current activated environment](#get-current-activated-environment) + * [Do something in environment without enabling it](#do-something-in-environment-without-enabling-it) + * [Get help](#get-help) + * [Get help for a command](#get-help-for-a-command) + * [How to add a plugin?](#how-to-add-a-plugin) + * [Mandatory elements](#mandatory-elements) + * [plug_list_versions](#plug_list_versions) + * [plug_url_for_download](#plug_url_for_download) + * [plug_build](#plug_build) + * [Optional elements](#optional-elements) + * [Variables](#variables) + * [Functions](#functions) + * [Examples](#examples) + * [Example of the usage](#example-of-the-usage) + * [Dependencies](#dependencies) + * [Supported OS](#supported-os) + * [Tests](#tests) + * [Version History](#version-history) + * [License](#license) + * [README in another language](#readme-in-another-language) +``` + +That's all! Now all you need — is copy/paste result from console into original +README.md. + +If you do not want to copy from console you can add `> YOURFILENAME.md` at the end of the command like `./gh-md-toc https://github.com/ekalinin/envirius/blob/master/README.md > table-of-contents.md` and this will store the table of contents to a file named table-of-contents.md in your current folder. + +And here is a result: + + * [README.md with TOC](https://github.com/ekalinin/envirius/blob/24ea3be0d3cc03f4235fa4879bb33dc122d0ae29/README.md) + +Moreover, it's able to work with GitHub's wiki pages: + +```bash +➥ ./gh-md-toc https://github.com/ekalinin/nodeenv/wiki/Who-Uses-Nodeenv + +Table of Contents +================= + + * [Who Uses Nodeenv?](#who-uses-nodeenv) + * [OpenStack](#openstack) + * [pre-commit.com](#pre-commitcom) +``` + +Multiple files +-------------- + +It supports multiple files as well: + +```bash +➥ ./gh-md-toc \ + https://github.com/aminb/rust-for-c/blob/master/hello_world/README.md \ + https://github.com/aminb/rust-for-c/blob/master/control_flow/README.md \ + https://github.com/aminb/rust-for-c/blob/master/primitive_types_and_operators/README.md \ + https://github.com/aminb/rust-for-c/blob/master/unique_pointers/README.md + + * [Hello world](https://github.com/aminb/rust-for-c/blob/master/hello_world/README.md#hello-world) + + * [Control Flow](https://github.com/aminb/rust-for-c/blob/master/control_flow/README.md#control-flow) + * [If](https://github.com/aminb/rust-for-c/blob/master/control_flow/README.md#if) + * [Loops](https://github.com/aminb/rust-for-c/blob/master/control_flow/README.md#loops) + * [For loops](https://github.com/aminb/rust-for-c/blob/master/control_flow/README.md#for-loops) + * [Switch/Match](https://github.com/aminb/rust-for-c/blob/master/control_flow/README.md#switchmatch) + * [Method call](https://github.com/aminb/rust-for-c/blob/master/control_flow/README.md#method-call) + + * [Primitive Types and Operators](https://github.com/aminb/rust-for-c/blob/master/primitive_types_and_operators/README.md#primitive-types-and-operators) + + * [Unique Pointers](https://github.com/aminb/rust-for-c/blob/master/unique_pointers/README.md#unique-pointers) +``` + +Combo +----- + +You can easily combine both ways: + +```bash +➥ ./gh-md-toc \ + ~/projects/Dockerfile.vim/README.md \ + https://github.com/ekalinin/sitemap.s/blob/master/README.md + + * [Dockerfile.vim](~/projects/Dockerfile.vim/README.md#dockerfilevim) + * [Screenshot](~/projects/Dockerfile.vim/README.md#screenshot) + * [Installation](~/projects/Dockerfile.vim/README.md#installation) + * [OR using Pathogen:](~/projects/Dockerfile.vim/README.md#or-using-pathogen) + * [OR using Vundle:](~/projects/Dockerfile.vim/README.md#or-using-vundle) + * [License](~/projects/Dockerfile.vim/README.md#license) + + * [sitemap.js](https://github.com/ekalinin/sitemap.js/blob/master/README.md#sitemapjs) + * [Installation](https://github.com/ekalinin/sitemap.js/blob/master/README.md#installation) + * [Usage](https://github.com/ekalinin/sitemap.js/blob/master/README.md#usage) + * [License](https://github.com/ekalinin/sitemap.js/blob/master/README.md#license) + + +``` + +Auto insert and update TOC +-------------------------- + +Just put into a file these two lines: + +``` + + +``` + +And run: + +```bash +$ ./gh-md-toc --insert README.test.md + +Table of Contents +================= + + * [gh-md-toc](#gh-md-toc) + * [Installation](#installation) + * [Usage](#usage) + * [STDIN](#stdin) + * [Local files](#local-files) + * [Remote files](#remote-files) + * [Multiple files](#multiple-files) + * [Combo](#combo) + * [Tests](#tests) + * [Dependency](#dependency) + +!! TOC was added into: 'README.test.md' +!! Origin version of the file: 'README.test.md.orig.2018-02-04_192655' +!! TOC added into a separate file: 'README.test.md.toc.2018-02-04_192655' + + + +``` + +Now check the same file: + +```bash +➜ grep -A15 "<\!\-\-ts" README.test.md + + * [gh-md-toc](#gh-md-toc) + * [Table of contents](#table-of-contents) + * [Installation](#installation) + * [Usage](#usage) + * [STDIN](#stdin) + * [Local files](#local-files) + * [Remote files](#remote-files) + * [Multiple files](#multiple-files) + * [Combo](#combo) + * [Auto insert and update TOC](#auto-insert-and-update-toc) + * [Tests](#tests) + * [Dependency](#dependency) + + + + +``` + +Next time when your file will be changed just repeat the command (`./gh-md-toc +--insert ...`) and TOC will be refreshed again. + +GitHub token +------------ + +All your tokens are [here](https://github.com/settings/tokens). + +You will need them if you get an error like this: + +``` +Parsing local markdown file requires access to github API +Error: You exceeded the hourly limit. See: https://developer.github.com/v3/#rate-limiting +or place github auth token here: ./token.txt +``` + +A token can be used as an env variable: + +```bash +➥ GH_TOC_TOKEN=2a2dab...563 ./gh-md-toc README.md + +Table of Contents +================= + +* [github\-markdown\-toc](#github-markdown-toc) +* [Table of Contents](#table-of-contents) +* [Installation](#installation) +* [Tests](#tests) +* [Usage](#usage) +* [LICENSE](#license) +``` + +Or from a file: + +```bash +➥ echo "2a2dab...563" > ./token.txt +➥ ./gh-md-toc README.md + +Table of Contents +================= + +* [github\-markdown\-toc](#github-markdown-toc) +* [Table of Contents](#table-of-contents) +* [Installation](#installation) +* [Tests](#tests) +* [Usage](#usage) +* [LICENSE](#license) +``` + +TOC generation with Github Actions +---------------------------------- + +Config: + +```yaml +on: + push: + branches: [main] + paths: ['foo.md'] + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v2 + - run: | + curl https://raw.githubusercontent.com/ekalinin/github-markdown-toc/0.8.0/gh-md-toc -o gh-md-toc + chmod a+x gh-md-toc + ./gh-md-toc --insert --no-backup --hide-footer foo.md + rm gh-md-toc + - uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Auto update markdown TOC +``` + +Tests +===== + +Done with [bats](https://github.com/bats-core/bats-core). +Useful articles: + + * https://www.engineyard.com/blog/how-to-use-bats-to-test-your-command-line-tools/ + * http://blog.spike.cx/post/60548255435/testing-bash-scripts-with-bats + + +How to run tests: + +```bash +➥ make test + + ✓ TOC for local README.md + ✓ TOC for remote README.md + ✓ TOC for mixed README.md (remote/local) + ✓ TOC for markdown from stdin + ✓ --help + ✓ --version + +6 tests, 0 failures +``` + +Dependency +========== + + * curl or wget + * awk (mawk is not tested) + * grep + * sed + * bats (for unit tests) + +Tested on Ubuntu 14.04/14.10 in bash/zsh. + +Docker +====== + +Local +----- + +* Build + +```shell +$ docker build -t markdown-toc-generator . +``` + +* Run on an URL + +```shell +$ docker run -it markdown-toc-generator https://github.com/ekalinin/envirius/blob/master/README.md +``` + +* Run on a local file (need to share volume with docker) + +```shell +$ docker run -it -v /data/ekalinin/envirius:/data markdown-toc-generator /data/README.md +``` + +Public +------- + +```shell +$ docker pull evkalinin/gh-md-toc:0.7.0 + +$ docker images | grep toc +evkalinin/gh-md-toc 0.7.0 0b8db6aed298 11 minutes ago 147MB + +$ docker run -it evkalinin/gh-md-toc:0.7.0 \ + https://github.com/ekalinin/envirius/blob/master/README.md +``` diff --git a/tools/scripts/md-toc/gh-md-toc b/tools/scripts/md-toc/gh-md-toc new file mode 100755 index 0000000000..35239bf5f8 --- /dev/null +++ b/tools/scripts/md-toc/gh-md-toc @@ -0,0 +1,421 @@ +#!/usr/bin/env bash + +# +# Steps: +# +# 1. Download corresponding html file for some README.md: +# curl -s $1 +# +# 2. Discard rows where no substring 'user-content-' (github's markup): +# awk '/user-content-/ { ... +# +# 3.1 Get last number in each row like ' ... sitemap.js.*<\/h/)+2, RLENGTH-5) +# +# 5. Find anchor and insert it inside "(...)": +# substr($0, match($0, "href=\"[^\"]+?\" ")+6, RLENGTH-8) +# + +gh_toc_version="0.10.0" + +gh_user_agent="gh-md-toc v$gh_toc_version" + +# +# Download rendered into html README.md by its url. +# +# +gh_toc_load() { + local gh_url=$1 + + if type curl &>/dev/null; then + curl --user-agent "$gh_user_agent" -s "$gh_url" + elif type wget &>/dev/null; then + wget --user-agent="$gh_user_agent" -qO- "$gh_url" + else + echo "Please, install 'curl' or 'wget' and try again." + exit 1 + fi +} + +# +# Converts local md file into html by GitHub +# +# -> curl -X POST --data '{"text": "Hello world github/linguist#1 **cool**, and #1!"}' https://api.github.com/markdown +#

Hello world github/linguist#1 cool, and #1!

'" +gh_toc_md2html() { + local gh_file_md=$1 + local skip_header=$2 + + URL=https://api.github.com/markdown/raw + + if [ -n "$GH_TOC_TOKEN" ]; then + TOKEN=$GH_TOC_TOKEN + else + TOKEN_FILE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/token.txt" + if [ -f "$TOKEN_FILE" ]; then + TOKEN="$(cat "$TOKEN_FILE")" + fi + fi + if [ -n "${TOKEN}" ]; then + AUTHORIZATION="Authorization: token ${TOKEN}" + fi + + local gh_tmp_file_md=$gh_file_md + if [ "$skip_header" = "yes" ]; then + if grep -Fxq "" "$gh_src"; then + # cut everything before the toc + gh_tmp_file_md=$gh_file_md~~ + sed '1,//d' "$gh_file_md" > "$gh_tmp_file_md" + fi + fi + + # echo $URL 1>&2 + OUTPUT=$(curl -s \ + --user-agent "$gh_user_agent" \ + --data-binary @"$gh_tmp_file_md" \ + -H "Content-Type:text/plain" \ + -H "$AUTHORIZATION" \ + "$URL") + + rm -f "${gh_file_md}~~" + + if [ "$?" != "0" ]; then + echo "XXNetworkErrorXX" + fi + if [ "$(echo "${OUTPUT}" | awk '/API rate limit exceeded/')" != "" ]; then + echo "XXRateLimitXX" + else + echo "${OUTPUT}" + fi +} + + +# +# Is passed string url +# +gh_is_url() { + case $1 in + https* | http*) + echo "yes";; + *) + echo "no";; + esac +} + +# +# TOC generator +# +gh_toc(){ + local gh_src=$1 + local gh_src_copy=$1 + local gh_ttl_docs=$2 + local need_replace=$3 + local no_backup=$4 + local no_footer=$5 + local indent=$6 + local skip_header=$7 + + if [ "$gh_src" = "" ]; then + echo "Please, enter URL or local path for a README.md" + exit 1 + fi + + + # Show "TOC" string only if working with one document + if [ "$gh_ttl_docs" = "1" ]; then + + echo "Table of Contents" + echo "=================" + echo "" + gh_src_copy="" + + fi + + if [ "$(gh_is_url "$gh_src")" == "yes" ]; then + gh_toc_load "$gh_src" | gh_toc_grab "$gh_src_copy" "$indent" + if [ "${PIPESTATUS[0]}" != "0" ]; then + echo "Could not load remote document." + echo "Please check your url or network connectivity" + exit 1 + fi + if [ "$need_replace" = "yes" ]; then + echo + echo "!! '$gh_src' is not a local file" + echo "!! Can't insert the TOC into it." + echo + fi + else + local rawhtml + rawhtml=$(gh_toc_md2html "$gh_src" "$skip_header") + if [ "$rawhtml" == "XXNetworkErrorXX" ]; then + echo "Parsing local markdown file requires access to github API" + echo "Please make sure curl is installed and check your network connectivity" + exit 1 + fi + if [ "$rawhtml" == "XXRateLimitXX" ]; then + echo "Parsing local markdown file requires access to github API" + echo "Error: You exceeded the hourly limit. See: https://developer.github.com/v3/#rate-limiting" + TOKEN_FILE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/token.txt" + echo "or place GitHub auth token here: ${TOKEN_FILE}" + exit 1 + fi + local toc + toc=`echo "$rawhtml" | gh_toc_grab "$gh_src_copy" "$indent"` + echo "$toc" + if [ "$need_replace" = "yes" ]; then + if grep -Fxq "" "$gh_src" && grep -Fxq "" "$gh_src"; then + echo "Found markers" + else + echo "You don't have or in your file...exiting" + exit 1 + fi + local ts="<\!--ts-->" + local te="<\!--te-->" + local dt + dt=$(date +'%F_%H%M%S') + local ext=".orig.${dt}" + local toc_path="${gh_src}.toc.${dt}" + local toc_createdby="" + local toc_footer + toc_footer="" + # http://fahdshariff.blogspot.ru/2012/12/sed-mutli-line-replacement-between-two.html + # clear old TOC + sed -i"${ext}" "/${ts}/,/${te}/{//!d;}" "$gh_src" + # create toc file + echo "${toc}" > "${toc_path}" + if [ "${no_footer}" != "yes" ]; then + echo -e "\n${toc_createdby}\n${toc_footer}\n" >> "$toc_path" + fi + + # insert toc file + if ! sed --version > /dev/null 2>&1; then + sed -i "" "/${ts}/r ${toc_path}" "$gh_src" + else + sed -i "/${ts}/r ${toc_path}" "$gh_src" + fi + echo + if [ "${no_backup}" = "yes" ]; then + rm "$toc_path" "$gh_src$ext" + fi + echo "!! TOC was added into: '$gh_src'" + if [ -z "${no_backup}" ]; then + echo "!! Origin version of the file: '${gh_src}${ext}'" + echo "!! TOC added into a separate file: '${toc_path}'" + fi + echo + fi + fi +} + +# +# Grabber of the TOC from rendered html +# +# $1 - a source url of document. +# It's need if TOC is generated for multiple documents. +# $2 - number of spaces used to indent. +# +gh_toc_grab() { + + href_regex="/href=\"[^\"]+?\"/" + common_awk_script=' + modified_href = "" + split(href, chars, "") + for (i=1;i <= length(href); i++) { + c = chars[i] + res = "" + if (c == "+") { + res = " " + } else { + if (c == "%") { + res = "\\x" + } else { + res = c "" + } + } + modified_href = modified_href res + } + print sprintf("%*s", (level-1)*'"$2"', "") "* [" text "](" gh_url modified_href ")" + ' + if [ "`uname -s`" == "OS/390" ]; then + grepcmd="pcregrep -o" + echoargs="" + awkscript='{ + level = substr($0, 3, 1) + text = substr($0, match($0, /<\/span><\/a>[^<]*<\/h/)+11, RLENGTH-14) + href = substr($0, match($0, '$href_regex')+6, RLENGTH-7) + '"$common_awk_script"' + }' + else + grepcmd="grep -Eo" + echoargs="-e" + awkscript='{ + level = substr($0, 3, 1) + text = substr($0, match($0, /">.*<\/h/)+2, RLENGTH-5) + href = substr($0, match($0, '$href_regex')+6, RLENGTH-7) + '"$common_awk_script"' + }' + fi + + # if closed is on the new line, then move it on the prev line + # for example: + # was: The command foo1 + # + # became: The command foo1 + sed -e ':a' -e 'N' -e '$!ba' -e 's/\n<\/h/<\/h/g' | + + # Sometimes a line can start with . Fix that. + sed -e ':a' -e 'N' -e '$!ba' -e 's/\n//g' | sed 's/<\/code>//g' | + + # remove g-emoji + sed 's/]*[^<]*<\/g-emoji> //g' | + + # now all rows are like: + #

title

.. + # format result line + # * $0 - whole string + # * last element of each row: "/dev/null; then + $tool --version | head -n 1 + else + echo "not installed" + fi + done +} + +show_help() { + local app_name + app_name=$(basename "$0") + echo "GitHub TOC generator ($app_name): $gh_toc_version" + echo "" + echo "Usage:" + echo " $app_name [options] src [src] Create TOC for a README file (url or local path)" + echo " $app_name - Create TOC for markdown from STDIN" + echo " $app_name --help Show help" + echo " $app_name --version Show version" + echo "" + echo "Options:" + echo " --indent Set indent size. Default: 3." + echo " --insert Insert new TOC into original file. For local files only. Default: false." + echo " See https://github.com/ekalinin/github-markdown-toc/issues/41 for details." + echo " --no-backup Remove backup file. Set --insert as well. Default: false." + echo " --hide-footer Do not write date & author of the last TOC update. Set --insert as well. Default: false." + echo " --skip-header Hide entry of the topmost headlines. Default: false." + echo " See https://github.com/ekalinin/github-markdown-toc/issues/125 for details." + echo "" +} + +# +# Options handlers +# +gh_toc_app() { + local need_replace="no" + local indent=3 + + if [ "$1" = '--help' ] || [ $# -eq 0 ] ; then + show_help + return + fi + + if [ "$1" = '--version' ]; then + show_version + return + fi + + if [ "$1" = '--indent' ]; then + indent="$2" + shift 2 + fi + + if [ "$1" = "-" ]; then + if [ -z "$TMPDIR" ]; then + TMPDIR="/tmp" + elif [ -n "$TMPDIR" ] && [ ! -d "$TMPDIR" ]; then + mkdir -p "$TMPDIR" + fi + local gh_tmp_md + if [ "`uname -s`" == "OS/390" ]; then + local timestamp + timestamp=$(date +%m%d%Y%H%M%S) + gh_tmp_md="$TMPDIR/tmp.$timestamp" + else + gh_tmp_md=$(mktemp "$TMPDIR/tmp.XXXXXX") + fi + while read -r input; do + echo "$input" >> "$gh_tmp_md" + done + gh_toc_md2html "$gh_tmp_md" | gh_toc_grab "" "$indent" + return + fi + + if [ "$1" = '--insert' ]; then + need_replace="yes" + shift + fi + + if [ "$1" = '--no-backup' ]; then + need_replace="yes" + no_backup="yes" + shift + fi + + if [ "$1" = '--hide-footer' ]; then + need_replace="yes" + no_footer="yes" + shift + fi + + if [ "$1" = '--skip-header' ]; then + skip_header="yes" + shift + fi + + + for md in "$@" + do + echo "" + gh_toc "$md" "$#" "$need_replace" "$no_backup" "$no_footer" "$indent" "$skip_header" + done + + echo "" + echo "" +} + +# +# Entry point +# +gh_toc_app "$@"