This is the initial commit. See description for what currently works

Currently Working:
* creating files
* deleting files
* editing files
* viewing files
* searching by id
* git
    - Initialises repository
    - commits when a file is created or edited (except rename)
    - arbitrary git commands on repo with `kb git ....`
* sql
    - Database is kept up to date when files are added, deleted or
      edited

TODO:
* Recreate database if deleted or freshly cloned
* Add database to gitignore if not in normal location
* Update database if edited without this tool
* Create a git commit if files are deleted or renamed
* makefile for install and test (currently just surecheck)
master
Jonathan Hodgson 3 years ago
commit 8ddda7f2b4
  1. 40
      README.md
  2. 647
      kb

@ -0,0 +1,40 @@
# Knowledge Base
This is a script that I use to manage my personal knowledge base. I have yet to
find a tool that fits my requirements / desires so I decided to build one.
This is still in early stages of development so expect braking changes to come
if you use it.
## Goals
### Mostly Plain Text
Most of my notes are currently in Markdown. This has a couple of advantages for
me:
* I can read them anywhere
* I can version control them with Git
There may be some exceptions. I may wish to include links or images which I will
version control with Git LFS; but for the most part, my notes are plain text.
### Tags
Before starting this project, my notes were organised into folders.
Unfortunately, this makes storing articles or notes that apply to different
areas difficult. I would prefer a tag based system. This would allow a file to
have multiple tags assigned to it.
### Fast
I want to be able to retrieve my notes quickly by tag or by title. To do this,
this tool will index notes using an SQLite database. This will not be version
controlled.
### Don't re-invent the wheel
I will be building on top of already great, fast tools such as
[RipGrep](https://github.com/BurntSushi/ripgrep) and
[FZF](https://github.com/junegunn/fzf).

647
kb

@ -0,0 +1,647 @@
#!/usr/bin/env bash
# Dependencies
# * bash (includes bash extentions that are not in posix shell eg [[ ]]
# * sqlite3
# * sed
# * awk
# * tr
# * git
# * yq (https://github.com/kislyuk/yq)
# * fzf (https://github.com/junegunn/fzf)
# * rg (https://github.com/BurntSushi/ripgrep)
# * bat (https://github.com/sharkdp/bat)
# Colour definitions
RED='\033[1;31m'
GREEN='\033[1;32m'
YELLOW='\033[1;33m'
NC='\033[0m'
# Provide a variable with the location of this script.
#scriptPath="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Utility functions
# ##################################################
die(){
necho -e "${RED}$*${NC}" >&2
exit 1
}
# Normal echo
# Echo if quiet is 0
necho(){
if [ "$quiet" -eq 0 ]; then
echo "$@"
fi
}
# verbose echo
# Echo if verbose is 1
vecho(){
if [ "$verbose" -eq 1 ]; then
echo "$@"
fi
}
# Exits after any cleanup that may be needed
safeExit(){
# TODO: Add cleanup
exit 0
}
isInt() {
[ "$1" -eq "$1" ] 2> /dev/null
return "$?"
}
# Set Flags
# -----------------------------------
# Flags which can be overridden by user input.
# Default values are below
# -----------------------------------
quiet=0
verbose=0
debug=0
dogit=1
args=()
dataDir="${XDG_DATA_HOME:=$HOME/.local/share}/kb/"
sqliteFile=""
editor="${EDITOR:=vim}"
# Check for Dependencies
# -----------------------------------
# Arrays containing package dependencies needed to execute this script.
# The script will fail if dependencies are not installed.
# -----------------------------------
checkDependencies(){
local dependencies=(
"sqlite3"
"rg"
"fzf"
"sed"
"awk"
"tr"
"git"
"yq"
"bat"
)
local ret=0
for program in "${dependencies[@]}"; do
if ! type "$program" &>/dev/null; then
necho -e "${RED}$program is not installed${NC}"
ret=1
case "$program" in
"rg")
necho -e "\t${YELLOW}This provides effecient searching${NC}"
;;
"fzf")
necho -e "\t${YELLOW}This provides fuzzy searching${NC}"
;;
esac
fi
done
return $ret
}
initKnowledgeBase(){
local output
necho -e "${YELLOW}Initialising Knowledge base${NC}"
vecho "Directory: $dataDir"
if [ "$verbose" -gt 0 ]; then
output="/dev/stdout"
else
output="/dev/null"
fi
[ -e "$dataDir" ] && die "$dataDir already exists"
mkdir -p "$dataDir"
if [ "$dogit" -gt 0 ]; then
git init "$dataDir" > "$output"
# TODO: make gitignore use new sqlite file
echo "/knowledgebase.sqlite3" >> "${dataDir}/.gitignore"
git -C "$dataDir" add .gitignore > "$output"
git -C "$dataDir" commit -m "Knowledge base initialised" > output
fi
vecho "Creating Database"
echo 'CREATE TABLE items
(id integer primary key, filename text, title text, type text);
CREATE TABLE tags
(id integer primary key, name text);
CREATE TABLE links
(id integer primary key, itemID integer, tagID integer); ' |
sqlite3 "${sqliteFile}"
necho -e "${GREEN}Initialised Knowledge base${NC}"
}
# This will create a fresh database based on the files in the folder
makedb(){
[ -f "$sqliteFile" ] && die "sqlite file already exists\ndelete it first if you want to"
echo "Still need to implement this"
}
getYamlBlock(){
vecho "getYamlBlock $*"
cd "$dataDir" || return
local filename
filename="$(findFile "$1")"
sed -n '1 { /^---/ { :a N; /\n---/! ba; p} }' "$filename" |
sed '1d;$d;s/\t/ /g'
}
getYamlTitle(){
vecho "getYamlTitle $*"
cd "$dataDir" || return
getYamlBlock "$1" | yq -r '.Title'
}
getYamlTags(){
vecho "getYamlTitle $*"
cd "$dataDir" || return
getYamlBlock "$1" | yq -r '.Tags | join("\n")'
}
# Makes file names safe
escapeFilename(){
vecho "escapeFilename $*"
echo "$1" |
tr ' ' '_' | # replace spaces with underscores
tr -d '/' # Delete slashes
}
findFileId(){
local filename
filename="$(findFile "$1")"
[ ! -e "$filename" ] && exit 1
echo "SELECT id FROM items WHERE filename = '$(safeSQL "$filename")'" |
sqlite3 "${sqliteFile}"
}
# Escapes ' and \ characters
safeSQL(){
echo "$1" |
sed 's/\\/\\\\/g' |
sed "s/'/\\\'/g"
}
assignTags(){
local filename
local tags
local tagIDs
local tagIDsOr
local fileID
filename="$(findFile "$1")"
[ ! -e "$filename" ] && exit 1
tags="$(cat - | sed '/^$/d')"
fileID="$(findFileId "$filename")"
# If there are tags
if [ -n "$tags" ]; then
local values
local orlist
while read -r line; do
values+=",('$(safeSQL "$line")')"
orlist+=" OR name = '$(safeSQL "$line")'"
done <<<"$(echo "$tags")"
values="$(echo "$values" | sed 's/^,//')"
orlist="$(echo "$orlist" | sed 's/^ OR //')"
# Ensure that all the tags exist
echo "INSERT INTO tags (name) VALUES $values
EXCEPT SELECT name FROM tags;" |
sqlite3 "${sqliteFile}"
# Get the tag ids we need to assosiate with the current file
tagIDs="$(echo "SELECT id FROM tags WHERE $orlist" |
sqlite3 "${sqliteFile}")"
#Loop through them all
while read -r tagID; do
tagIDsOr+=" OR tagID = $(safeSQL "$tagID")"
# Check the tag is already linkded with the file
local existing
existing="$(echo "SELECT id FROM links
WHERE itemID = $(safeSQL "$fileID")
AND tagID = $(safeSQL "$tagID")" |
sqlite3 "${sqliteFile}"
)"
# If not, add a link
if [ -z "$existing" ]; then
echo "INSERT INTO links (itemID,tagID)
VALUES ($(safeSQL "$fileID"),$(safeSQL "$tagID"))" |
sqlite3 "${sqliteFile}"
fi
done <<<"$(echo "$tagIDs")"
tagIDsOr="$(echo "$tagIDsOr" | sed 's/^ OR //')"
# Delete any links that are not in the list
echo "DELETE FROM links
WHERE itemID = $(safeSQL "$fileID")
AND NOT ( $tagIDsOr )" |
sqlite3 "${sqliteFile}"
else # If there are no tags, simply delete any that are referenced
echo "DELETE FROM links WHERE itemID = '$(safeSQL "$fileID")'" |
sqlite3 "${sqliteFile}"
fi
}
newFile(){
vecho "newFile $*"
cd "$dataDir" || return
# While there is a - at the begining
local title="$*"
if [ -z "$title" ]; then
echo -n "Enter a title: "
read -r title
fi
local filename
filename="$(escapeFilename "$title.md")"
[ -e "$filename" ] && die "$filename already exists"
echo -e "---
Title: $title
Tags:
-
---
" > "$filename"
echo "INSERT INTO items (filename, title, type)
VALUES ( '$(safeSQL "$filename")', '$(safeSQL "$title")', 'normal' );" |
sqlite3 "${sqliteFile}"
editFile "$filename"
}
findFile(){
vecho "findFile $*"
cd "$dataDir" || return
local filename
if [ "$#" -eq 1 ] && isInt "$1"; then
filename="$(echo "SELECT filename
FROM items WHERE id = '$(safeSQL "$1")'" |
sqlite3 "${sqliteFile}")"
fi
[ -z "$filename" ] && filename="$(escapeFilename "$*")"
if [ -e "$filename" ]; then
# If the file exists, return it
echo "$filename"
exit 0
else
# If it exists with .md at the end, assume that was meant
# otherwise die
if [ -e "$filename.md" ]; then
echo "$filename.md"
exit 0
else
die "No such file or ID $filename"
exit 1
fi
fi
}
dogit(){
cd "$dataDir" || return
git "$@"
}
# This function will add and commit a file after it has been edited
gitChange(){
cd "$dataDir" || return
local filename="$1"
if [ "$dogit" -gt 0 ]; then
if [ -f "$filename" ]; then
if ! git diff --exit-code "$filename" > /dev/null 2>&1; then
# Changes
git add "$filename"
git commit -m "KB auto-commit: Updated: $filename"
elif ! git ls-files --error-unmatch "$filename" > /dev/null 2>&1; then
# New file
git add "$filename"
git commit -m "KB auto-commit: New: $filename"
fi
fi
fi
}
# Takes the filename as a parameter
editFile(){
vecho "editFile $*"
cd "$dataDir" || return
local filename
local oldTitle
local newTitle
filename="$(findFile "$*")"
[ ! -e "$filename" ] && exit 1
oldTitle="$(getYamlTitle "$filename")"
"$editor" "$filename"
newTitle="$(getYamlTitle "$filename")"
getYamlTags "$filename" | assignTags "$filename"
if [ "$newTitle" != "$oldTitle" ]; then
vecho "Changed title"
local newfilename
newfilename="$(escapeFilename "$newTitle.md")"
if [ -e "$newfilename" ]; then
echo -e "${YELLOW}File name $newfilename already exists${NC}"
echo -e "Please fix manually"
exit 1
else
mv "$filename" "$newfilename"
echo "UPDATE items
SET (filename,title) = ('$(safeSQL "$newfilename")','$(safeSQL "$newTitle")')
WHERE filename = '$(safeSQL "$filename")';" |
sqlite3 "${sqliteFile}"
gitChange "$newfilename"
fi
else
gitChange "$filename"
fi
}
listEntries(){
vecho "listEntries $*"
cd "$dataDir" || return
echo "SELECT items.id,items.filename,items.title,items.type,
GROUP_CONCAT(tags.name,',') tags
FROM items LEFT JOIN links ON items.id = links.itemID
LEFT JOIN tags ON links.tagID = tags.id
GROUP BY items.id;" |
sqlite3 --column --header "${sqliteFile}"
}
fuzzySelect(){
cd "$dataDir" || return
local id
export -f fzfPreview
export dataDir
id="$(listEntries | fzf --header-lines=2 --delimiter=" +" --with-nth=3,5 \
--preview='bash -c "fzfPreview {}"' | awk '{print $1}')"
if [ -n "$id" ]; then
case "$1" in
edit) editFile "$id"; safeExit ;;
view) viewFile "$id"; safeExit ;;
esac
fi
}
fzfPreview(){
cd "$dataDir" || return
local id
local filename
local title
local type
local tags
id="$(echo "$1" | awk -F ' +' '{print $1}')"
filename="$(echo "$1" | awk -F ' +' '{print $2}')"
title="$(echo "$1" | awk -F ' +' '{print $3}')"
type="$(echo "$1" | awk -F ' +' '{print $4}')"
tags="$(echo "$1" | awk -F ' +' '{print $5}')"
if [ "$type" = "normal" ]; then
bat --color=always --style=numbers "$filename"
fi
}
viewFile(){
cd "$dataDir" || return
local id="$1"
local filename
filename="$(findFile "$id")"
bat --color=always --style=full "$filename"
}
deleteFile(){
cd "$dataDir" || return
local filename
local fileID
local rsp
filename="$(findFile "$1")"
fileID="$(findFileId "$filename")"
[ ! -e "$filename" ] && exit 1
echo -n "Are you sure? [yN] "
read -r rsp
if [[ "$(echo "$rsp" | tr '[:upper:]' '[:lower:]')" = "y"* ]]; then
rm "$filename"
# This deletes the file from the sql database and any tag links
echo "DELETE FROM items
WHERE id = '$(safeSQL "$fileID")';
DELETE FROM links
WHERE itemID = '$(safeSQL "$fileID")';" |
sqlite3 --column --header "${sqliteFile}"
fi
}
doDeepSearch(){
cd "$dataDir" || return
local query="$1"
echo "$query"
rg --column --line-number --no-heading --color=always --smart-case "$query"
}
doDeepPreview(){
cd "$dataDir" || return
local file
local line
file="$(echo "$1" | cut -d ':' -f 1)"
line="$(echo "$1" | cut -d ':' -f 2)"
bat --color=always --style=numbers -H "$line" "$file"
}
deepSearch(){
type -p rg > /dev/null || die "You need rg installed for deep search"
export -f doDeepSearch
export -f doDeepPreview
export dataDir
echo "" | fzf --ansi \
--bind 'change:reload:bash -c "doDeepSearch {q} || true"' \
--preview 'bash -c "doDeepPreview {} || true"'
}
mainScript() {
############## Begin Script Here ###################
####################################################
if [ "${args[0]}" != "init" ]; then
cd "$dataDir" || return
# Check to see if datadir is a git repo
if git rev-parse 2> /dev/null; then
# If not, ensure we don't run git commands
dogit=0
fi
fi
case "${args[0]}" in
init) initKnowledgeBase; safeExit ;;
makedb) makedb; safeExit ;;
new) newFile "${args[@]:1}"; safeExit ;;
edit) editFile "${args[@]:1}"; safeExit ;;
list) listEntries "${args[@]:1}"; safeExit ;;
view) viewFile "${args[@]:1}"; safeExit ;;
fuzzy) fuzzySelect "${args[@]:1}"; safeExit ;;
deepsearch) deepSearch "${args[@]:1}"; safeExit ;;
del|delete) deleteFile "${args[@]:1}"; safeExit ;;
git) dogit "${args[@]:1}"; safeExit ;;
*) usage >&2; safeExit ;;
esac
####################################################
############### End Script Here ####################
}
############## Begin Options and Usage ###################
# Print usage
usage() {
echo -n "kb [OPTIONS]... COMMAND
Knowledge Base Management
This tool helps manage my personal knowledge base
Options:
-q, --quiet Quiet (no output)
-v, --verbose Output more information. (Items echoed to 'verbose')
-d, --debug Runs script in BASH debug mode (set -x)
-h, --help Display this help and exit
--data <directory> The knowledgebase data dir
--sqlite <file> The sqlite file (default to <data>/knowledgebase.sqlite3
--nogit Don't run git commands
--version Output version information and exit
Commands:
init Initialise a new knowledgebase
new [options] [title] Create new entry
--filetype <type> Type of file ( default md)
edit [title] Edit a file
list List all files
view View a file
fuzzy [command] Fuzzy select a file
command is what to do with the selected file
edit or view
git [options] Run arbitary git commands on the kb repository
del [title] Delete a file
"
}
# Iterate over options breaking -ab into -a -b when needed and --foo=bar into
# --foo bar
optstring=h
unset options
while (($#)); do
case $1 in
# If option is of type -ab
-[!-]?*)
# Loop over each character starting with the second
for ((i=1; i < ${#1}; i++)); do
c=${1:i:1}
# Add current char to options
options+=("-$c")
# If option takes a required argument, and it's not the last char make
# the rest of the string its argument
if [[ $optstring = *"$c:"* && ${1:i+1} ]]; then
options+=("${1:i+1}")
break
fi
done
;;
# If option is of type --foo=bar
--?*=*) options+=("${1%%=*}" "${1#*=}") ;;
# add --endopts for --
--) options+=(--endopts) ;;
# Otherwise, nothing special
*) options+=("$1") ;;
esac
shift
done
set -- "${options[@]}"
unset options
# Print help if no arguments were passed.
# Uncomment to force arguments when invoking the script
[[ $# -eq 0 ]] && set -- "--help"
# Read the options and set stuff
while [[ $1 = -?* ]]; do
case $1 in
-h|--help) usage >&2; safeExit ;;
-v|--verbose) verbose=1 ;;
-q|--quiet) quiet=1 ;;
-d|--debug) debug=1;;
--data) dataDir="$2"; shift ;;
--sqlite) sqliteFile="$2"; shift ;;
--editor) editor="$2"; shift ;;
--nogit) dogit=0 ;;
--) shift; break ;;
*) die "invalid option: '$1'." ;;
esac
shift
done
# If the sqlite file hasn't been specified on the command line, make it a file
# called knowledgebase.sqlite3 in the dataDir
[ -z "$sqliteFile" ] && sqliteFile="${dataDir}/knowledgebase.sqlite3"
# Store the remaining part as arguments.
args+=("$@")
############## End Options and Usage ###################
# ############# ############# #############
# ## TIME TO RUN THE SCRIPT ##
# ## ##
# ## You shouldn't need to edit anything ##
# ## beneath this line ##
# ## ##
# ############# ############# #############
# Exit on error. Append '||true' when you run the script if you expect an error.
set -o errexit
# Run in debug mode, if set
if [ "${debug}" == "1" ]; then
set -x
fi
# Bash will remember & return the highest exitcode in a chain of pipes.
# This way you can catch the error in case mysqldump fails in `mysqldump |gzip`, for example.
set -o pipefail
# Invoke the checkDependencies function to test for Bash packages
checkDependencies
# Run your script
mainScript
safeExit # Exit cleanly
[ -n "$KB_DIR" ] && dataDir="$KB_DIR" ||
dataDir="${XDG_DATA_HOME:=$HOME/.local/share}/kb/"
mkdir -p "$dataDir"
Loading…
Cancel
Save