#!/usr/bin/env bash # Dependencies # * bash (includes bash extentions that are not in posix shell eg [[ ]] # * sqlite3 # * sed # * awk # * tr # * git # * yq (https://github.com/kislyuk/yq) # * fzf (https://github.com/junegunn/fzf) # * rg (https://github.com/BurntSushi/ripgrep) # * bat (https://github.com/sharkdp/bat) # Colour definitions RED='\033[1;31m' GREEN='\033[1;32m' YELLOW='\033[1;33m' NC='\033[0m' # Provide a variable with the location of this script. #scriptPath="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # Utility functions # ################################################## die(){ necho -e "${RED}$*${NC}" >&2 exit 1 } # Normal echo # Echo if quiet is 0 necho(){ if [ "$quiet" -eq 0 ]; then echo "$@" fi } # verbose echo # Echo if verbose is 1 vecho(){ if [ "$verbose" -eq 1 ]; then echo "$@" fi } # Exits after any cleanup that may be needed safeExit(){ # TODO: Add cleanup exit 0 } isInt() { [ "$1" -eq "$1" ] 2> /dev/null return "$?" } # Set Flags # ----------------------------------- # Flags which can be overridden by user input. # Default values are below # ----------------------------------- quiet=0 verbose=0 debug=0 dogit=1 args=() dataDir="${XDG_DATA_HOME:=$HOME/.local/share}/kb/" sqliteFile="" editor="${EDITOR:=vim}" # Check for Dependencies # ----------------------------------- # Arrays containing package dependencies needed to execute this script. # The script will fail if dependencies are not installed. # ----------------------------------- checkDependencies(){ local dependencies=( "sqlite3" "rg" "fzf" "sed" "awk" "tr" "git" "yq" "bat" ) local ret=0 for program in "${dependencies[@]}"; do if ! type "$program" &>/dev/null; then necho -e "${RED}$program is not installed${NC}" ret=1 case "$program" in "rg") necho -e "\t${YELLOW}This provides effecient searching${NC}" ;; "fzf") necho -e "\t${YELLOW}This provides fuzzy searching${NC}" ;; esac fi done return $ret } initKnowledgeBase(){ local output necho -e "${YELLOW}Initialising Knowledge base${NC}" vecho "Directory: $dataDir" if [ "$verbose" -gt 0 ]; then output="/dev/stdout" else output="/dev/null" fi [ -e "$dataDir" ] && die "$dataDir already exists" mkdir -p "$dataDir" if [ "$dogit" -gt 0 ]; then git init "$dataDir" > "$output" # TODO: make gitignore use new sqlite file echo "/knowledgebase.sqlite3" >> "${dataDir}/.gitignore" git -C "$dataDir" add .gitignore > "$output" git -C "$dataDir" commit -m "Knowledge base initialised" > output fi vecho "Creating Database" echo 'CREATE TABLE items (id integer primary key, filename text, title text, type text); CREATE TABLE tags (id integer primary key, name text); CREATE TABLE links (id integer primary key, itemID integer, tagID integer); ' | sqlite3 "${sqliteFile}" necho -e "${GREEN}Initialised Knowledge base${NC}" } # This will create a fresh database based on the files in the folder makedb(){ [ -f "$sqliteFile" ] && die "sqlite file already exists\ndelete it first if you want to" echo "Still need to implement this" } getYamlBlock(){ vecho "getYamlBlock $*" cd "$dataDir" || return local filename filename="$(findFile "$1")" sed -n '1 { /^---/ { :a N; /\n---/! ba; p} }' "$filename" | sed '1d;$d;s/\t/ /g' } getYamlTitle(){ vecho "getYamlTitle $*" cd "$dataDir" || return getYamlBlock "$1" | yq -r '.Title' } getYamlTags(){ vecho "getYamlTitle $*" cd "$dataDir" || return getYamlBlock "$1" | yq -r '.Tags | join("\n")' } # Makes file names safe escapeFilename(){ vecho "escapeFilename $*" echo "$1" | tr ' ' '_' | # replace spaces with underscores tr -d '/' # Delete slashes } findFileId(){ local filename filename="$(findFile "$1")" [ ! -e "$filename" ] && exit 1 echo "SELECT id FROM items WHERE filename = '$(safeSQL "$filename")'" | sqlite3 "${sqliteFile}" } # Escapes ' and \ characters safeSQL(){ echo "$1" | sed 's/\\/\\\\/g' | sed "s/'/\\\'/g" } assignTags(){ local filename local tags local tagIDs local tagIDsOr local fileID filename="$(findFile "$1")" [ ! -e "$filename" ] && exit 1 tags="$(cat - | sed '/^$/d')" fileID="$(findFileId "$filename")" # If there are tags if [ -n "$tags" ]; then local values local orlist while read -r line; do values+=",('$(safeSQL "$line")')" orlist+=" OR name = '$(safeSQL "$line")'" done <<<"$(echo "$tags")" values="$(echo "$values" | sed 's/^,//')" orlist="$(echo "$orlist" | sed 's/^ OR //')" # Ensure that all the tags exist echo "INSERT INTO tags (name) VALUES $values EXCEPT SELECT name FROM tags;" | sqlite3 "${sqliteFile}" # Get the tag ids we need to assosiate with the current file tagIDs="$(echo "SELECT id FROM tags WHERE $orlist" | sqlite3 "${sqliteFile}")" #Loop through them all while read -r tagID; do tagIDsOr+=" OR tagID = $(safeSQL "$tagID")" # Check the tag is already linkded with the file local existing existing="$(echo "SELECT id FROM links WHERE itemID = $(safeSQL "$fileID") AND tagID = $(safeSQL "$tagID")" | sqlite3 "${sqliteFile}" )" # If not, add a link if [ -z "$existing" ]; then echo "INSERT INTO links (itemID,tagID) VALUES ($(safeSQL "$fileID"),$(safeSQL "$tagID"))" | sqlite3 "${sqliteFile}" fi done <<<"$(echo "$tagIDs")" tagIDsOr="$(echo "$tagIDsOr" | sed 's/^ OR //')" # Delete any links that are not in the list echo "DELETE FROM links WHERE itemID = $(safeSQL "$fileID") AND NOT ( $tagIDsOr )" | sqlite3 "${sqliteFile}" else # If there are no tags, simply delete any that are referenced echo "DELETE FROM links WHERE itemID = '$(safeSQL "$fileID")'" | sqlite3 "${sqliteFile}" fi } newFile(){ vecho "newFile $*" cd "$dataDir" || return # While there is a - at the begining local title="$*" if [ -z "$title" ]; then echo -n "Enter a title: " read -r title fi local filename filename="$(escapeFilename "$title.md")" [ -e "$filename" ] && die "$filename already exists" echo -e "--- Title: $title Tags: - --- " > "$filename" echo "INSERT INTO items (filename, title, type) VALUES ( '$(safeSQL "$filename")', '$(safeSQL "$title")', 'normal' );" | sqlite3 "${sqliteFile}" editFile "$filename" } findFile(){ vecho "findFile $*" cd "$dataDir" || return local filename if [ "$#" -eq 1 ] && isInt "$1"; then filename="$(echo "SELECT filename FROM items WHERE id = '$(safeSQL "$1")'" | sqlite3 "${sqliteFile}")" fi [ -z "$filename" ] && filename="$(escapeFilename "$*")" if [ -e "$filename" ]; then # If the file exists, return it echo "$filename" exit 0 else # If it exists with .md at the end, assume that was meant # otherwise die if [ -e "$filename.md" ]; then echo "$filename.md" exit 0 else die "No such file or ID $filename" exit 1 fi fi } dogit(){ cd "$dataDir" || return git "$@" } # This function will add and commit a file after it has been edited gitChange(){ cd "$dataDir" || return local filename="$1" if [ "$dogit" -gt 0 ]; then if [ -f "$filename" ]; then if ! git diff --exit-code "$filename" > /dev/null 2>&1; then # Changes git add "$filename" git commit -m "KB auto-commit: Updated: $filename" elif ! git ls-files --error-unmatch "$filename" > /dev/null 2>&1; then # New file git add "$filename" git commit -m "KB auto-commit: New: $filename" fi fi fi } # Takes the filename as a parameter editFile(){ vecho "editFile $*" cd "$dataDir" || return local filename local oldTitle local newTitle filename="$(findFile "$*")" [ ! -e "$filename" ] && exit 1 oldTitle="$(getYamlTitle "$filename")" "$editor" "$filename" newTitle="$(getYamlTitle "$filename")" getYamlTags "$filename" | assignTags "$filename" if [ "$newTitle" != "$oldTitle" ]; then vecho "Changed title" local newfilename newfilename="$(escapeFilename "$newTitle.md")" if [ -e "$newfilename" ]; then echo -e "${YELLOW}File name $newfilename already exists${NC}" echo -e "Please fix manually" exit 1 else mv "$filename" "$newfilename" echo "UPDATE items SET (filename,title) = ('$(safeSQL "$newfilename")','$(safeSQL "$newTitle")') WHERE filename = '$(safeSQL "$filename")';" | sqlite3 "${sqliteFile}" gitChange "$newfilename" fi else gitChange "$filename" fi } listEntries(){ vecho "listEntries $*" cd "$dataDir" || return echo "SELECT items.id,items.filename,items.title,items.type, GROUP_CONCAT(tags.name,',') tags FROM items LEFT JOIN links ON items.id = links.itemID LEFT JOIN tags ON links.tagID = tags.id GROUP BY items.id;" | sqlite3 --column --header "${sqliteFile}" } fuzzySelect(){ cd "$dataDir" || return local id export -f fzfPreview export dataDir id="$(listEntries | fzf --header-lines=2 --delimiter=" +" --with-nth=3,5 \ --preview='bash -c "fzfPreview {}"' | awk '{print $1}')" if [ -n "$id" ]; then case "$1" in edit) editFile "$id"; safeExit ;; view) viewFile "$id"; safeExit ;; esac fi } fzfPreview(){ cd "$dataDir" || return local id local filename local title local type local tags id="$(echo "$1" | awk -F ' +' '{print $1}')" filename="$(echo "$1" | awk -F ' +' '{print $2}')" title="$(echo "$1" | awk -F ' +' '{print $3}')" type="$(echo "$1" | awk -F ' +' '{print $4}')" tags="$(echo "$1" | awk -F ' +' '{print $5}')" if [ "$type" = "normal" ]; then bat --color=always --style=numbers "$filename" fi } viewFile(){ cd "$dataDir" || return local id="$1" local filename filename="$(findFile "$id")" bat --color=always --style=full "$filename" } deleteFile(){ cd "$dataDir" || return local filename local fileID local rsp filename="$(findFile "$1")" fileID="$(findFileId "$filename")" [ ! -e "$filename" ] && exit 1 echo -n "Are you sure? [yN] " read -r rsp if [[ "$(echo "$rsp" | tr '[:upper:]' '[:lower:]')" = "y"* ]]; then rm "$filename" # This deletes the file from the sql database and any tag links echo "DELETE FROM items WHERE id = '$(safeSQL "$fileID")'; DELETE FROM links WHERE itemID = '$(safeSQL "$fileID")';" | sqlite3 --column --header "${sqliteFile}" fi } doDeepSearch(){ cd "$dataDir" || return local query="$1" echo "$query" rg --column --line-number --no-heading --color=always --smart-case "$query" } doDeepPreview(){ cd "$dataDir" || return local file local line file="$(echo "$1" | cut -d ':' -f 1)" line="$(echo "$1" | cut -d ':' -f 2)" bat --color=always --style=numbers -H "$line" "$file" } deepSearch(){ type -p rg > /dev/null || die "You need rg installed for deep search" export -f doDeepSearch export -f doDeepPreview export dataDir echo "" | fzf --ansi \ --bind 'change:reload:bash -c "doDeepSearch {q} || true"' \ --preview 'bash -c "doDeepPreview {} || true"' } mainScript() { ############## Begin Script Here ################### #################################################### if [ "${args[0]}" != "init" ]; then cd "$dataDir" || return # Check to see if datadir is a git repo if git rev-parse 2> /dev/null; then # If not, ensure we don't run git commands dogit=0 fi fi case "${args[0]}" in init) initKnowledgeBase; safeExit ;; makedb) makedb; safeExit ;; new) newFile "${args[@]:1}"; safeExit ;; edit) editFile "${args[@]:1}"; safeExit ;; list) listEntries "${args[@]:1}"; safeExit ;; view) viewFile "${args[@]:1}"; safeExit ;; fuzzy) fuzzySelect "${args[@]:1}"; safeExit ;; deepsearch) deepSearch "${args[@]:1}"; safeExit ;; del|delete) deleteFile "${args[@]:1}"; safeExit ;; git) dogit "${args[@]:1}"; safeExit ;; *) usage >&2; safeExit ;; esac #################################################### ############### End Script Here #################### } ############## Begin Options and Usage ################### # Print usage usage() { echo -n "kb [OPTIONS]... COMMAND Knowledge Base Management This tool helps manage my personal knowledge base Options: -q, --quiet Quiet (no output) -v, --verbose Output more information. (Items echoed to 'verbose') -d, --debug Runs script in BASH debug mode (set -x) -h, --help Display this help and exit --data The knowledgebase data dir --sqlite The sqlite file (default to /knowledgebase.sqlite3 --nogit Don't run git commands --version Output version information and exit Commands: init Initialise a new knowledgebase new [options] [title] Create new entry --filetype Type of file ( default md) edit [title] Edit a file list List all files view View a file fuzzy [command] Fuzzy select a file command is what to do with the selected file edit or view git [options] Run arbitary git commands on the kb repository del [title] Delete a file " } # Iterate over options breaking -ab into -a -b when needed and --foo=bar into # --foo bar optstring=h unset options while (($#)); do case $1 in # If option is of type -ab -[!-]?*) # Loop over each character starting with the second for ((i=1; i < ${#1}; i++)); do c=${1:i:1} # Add current char to options options+=("-$c") # If option takes a required argument, and it's not the last char make # the rest of the string its argument if [[ $optstring = *"$c:"* && ${1:i+1} ]]; then options+=("${1:i+1}") break fi done ;; # If option is of type --foo=bar --?*=*) options+=("${1%%=*}" "${1#*=}") ;; # add --endopts for -- --) options+=(--endopts) ;; # Otherwise, nothing special *) options+=("$1") ;; esac shift done set -- "${options[@]}" unset options # Print help if no arguments were passed. # Uncomment to force arguments when invoking the script [[ $# -eq 0 ]] && set -- "--help" # Read the options and set stuff while [[ $1 = -?* ]]; do case $1 in -h|--help) usage >&2; safeExit ;; -v|--verbose) verbose=1 ;; -q|--quiet) quiet=1 ;; -d|--debug) debug=1;; --data) dataDir="$2"; shift ;; --sqlite) sqliteFile="$2"; shift ;; --editor) editor="$2"; shift ;; --nogit) dogit=0 ;; --) shift; break ;; *) die "invalid option: '$1'." ;; esac shift done # If the sqlite file hasn't been specified on the command line, make it a file # called knowledgebase.sqlite3 in the dataDir [ -z "$sqliteFile" ] && sqliteFile="${dataDir}/knowledgebase.sqlite3" # Store the remaining part as arguments. args+=("$@") ############## End Options and Usage ################### # ############# ############# ############# # ## TIME TO RUN THE SCRIPT ## # ## ## # ## You shouldn't need to edit anything ## # ## beneath this line ## # ## ## # ############# ############# ############# # Exit on error. Append '||true' when you run the script if you expect an error. set -o errexit # Run in debug mode, if set if [ "${debug}" == "1" ]; then set -x fi # Bash will remember & return the highest exitcode in a chain of pipes. # This way you can catch the error in case mysqldump fails in `mysqldump |gzip`, for example. set -o pipefail # Invoke the checkDependencies function to test for Bash packages checkDependencies # Run your script mainScript safeExit # Exit cleanly [ -n "$KB_DIR" ] && dataDir="$KB_DIR" || dataDir="${XDG_DATA_HOME:=$HOME/.local/share}/kb/" mkdir -p "$dataDir"