'" >&2
exit 1
fi
parsedModels=()
if [ -n "$modelsList" ]; then
IFS=',' read -ra modelsListArray <<< "$modelsList" # parse csv into modelsListArray
for m in "${modelsListArray[@]}"; do
if [[ "${models[*]}" =~ "$m" ]]; then # if model exists
parsedModels+=("$m")
else
echo "Error: model not found: $m" >&2
exit 1
fi
done
fi
if [ -n "$parsedModels" ]; then
models=("${parsedModels[@]}")
fi
echo "models:";
echo "${models[@]}"
echo
}
safeString() {
local input="$1" # Get the input
input=${input:0:42} # Truncate to first 42 characters
input=$(echo "$input" | tr '[:upper:]' '[:lower:]') # Convert to lowercase
input=$(echo "$input" | sed "s/ /_/g") # Replace spaces with underscores
input=$(echo "$input" | sed 's/[^a-zA-Z0-9_]/_/g' | tr -cd 'a-zA-Z0-9_') # Replace non-allowed characters with underscores
echo "$input" # Output the sanitized string
}
createOutputDirectory() {
tag=$(safeString "$prompt")
tagDatetime=$(date '+%Y%m%d-%H%M%S')
outputDirectory="$resultsDirectory/${tag}_${tagDatetime}"
echo $(getDateTime) "Output Directory: $outputDirectory/"
if [ ! -d "$outputDirectory" ]; then
if ! mkdir -p "$outputDirectory"; then
echo "Error: Failed to create Output Directory $outputDirectory" >&2
exit 1
fi
fi
}
setPrompt() {
if [ -n "$prompt" ]; then # if prompt is already set from command line
return
fi
if [ -t 0 ]; then # Check if input is from a terminal (interactive)
echo "Enter prompt:";
read -r prompt # Read prompt from user input
return
fi
prompt=$(cat) # Read from standard input (pipe or file)
}
savePrompt() {
echo $(getDateTime) "Prompt: $prompt"
promptFile="$outputDirectory/prompt.txt"
echo $(getDateTime) "Creating Prompt Text: $promptFile"
echo "$prompt" > "$promptFile"
promptWords=$(wc -w < "$promptFile" | awk '{print $1}')
promptBytes=$(wc -c < "$promptFile" | awk '{print $1}')
promptYamlFile="$outputDirectory/$tag.prompt.yaml"
echo $(getDateTime) "Creating Prompt Yaml: $promptYamlFile"
generatePromptYaml > "$promptYamlFile"
}
generatePromptYaml() {
# Github Prompt YAML: https://docs.github.com/en/github-models/use-github-models/storing-prompts-in-github-repositories
cat << EOF
messages:
- role: system
content: ''
- role: user
content: |
$(while IFS= read -r line; do echo " $line"; done <<< "$prompt")
model: ''
EOF
}
textarea() {
local content="$1" # Get the input
if [ -z "$content" ]; then
content=""
fi
local padding="$2"
if [ -z "$padding" ]; then
padding=0
fi
local max="$3"
if [ -z "$max" ]; then
max=25
fi
local lines=$(echo "$content\n" | wc -l) # Get number of lines in content
lines=$((lines + padding))
if [ "$lines" -gt "$max" ]; then
lines=$max
fi
content=$(echo "$content" | sed 's/&/\&/g; s/\</g; s/>/\>/g; s/"/\"/g; s/'"'"'/\'/g') # Escape HTML special characters
echo ""
}
showPrompt() {
echo "Prompt: (raw) (yaml)"
echo " words:$promptWords bytes:$promptBytes
"
textarea "$prompt" 2 10 # 0 padding, max 10 lines
echo "
"
}
showImages() {
if [ -n "$addedImages" ]; then
for image in ${addedImages}; do
echo -n ""
echo -n "
)
"
echo -n "
"
done
echo -n "
"
fi
}
clearModel() {
echo $(getDateTime) "Clearing model session: $1"
(
expect \
-c "spawn ollama run $1" \
-c "expect \">>> \"" \
-c 'send -- "/clear\n"' \
-c "expect \"Cleared session context\"" \
-c 'send -- "/bye\n"' \
-c "expect eof" \
;
) > /dev/null 2>&1 # Suppress output
if [ $? -ne 0 ]; then
echo "ERROR: Failed to clear model session: $1" >&2
# exit 1
fi
}
stopModel() {
echo $(getDateTime) "Stopping model: $1"
ollama stop "$1"
if [ $? -ne 0 ]; then
echo "ERROR: Failed to stop model: $1" >&2
# exit 1
fi
}
showSortableTablesJavascript() {
# From: https://github.com/tofsjonas/sortable/
# License: The Unlicense - https://github.com/tofsjonas/sortable/blob/main/LICENSE
echo ''
echo ''
}
showHeader() {
title="$1"
cat << "EOF"
EOF
echo "$title"
}
showFooter() {
title="$1"
echo "
"
echo ""
}
createMenu() {
local currentModel="$1"
echo "';
}
setStats() {
statsTotalDuration=$(grep -oE "total duration:[[:space:]]+(.*)" "$modelStatsTxt" | awk '{ print $NF }')
statsLoadDuration=$(grep -oE "load duration:[[:space:]]+(.*)" "$modelStatsTxt" | awk '{ print $NF }')
statsPromptEvalCount=$(grep -oE "prompt eval count:[[:space:]]+(.*)" "$modelStatsTxt" | awk '{ print $4, $5 }')
statsPromptEvalDuration=$(grep -oE "prompt eval duration:[[:space:]]+(.*)" "$modelStatsTxt" | awk '{ print $NF }')
statsPromptEvalRate=$(grep -oE "prompt eval rate:[[:space:]]+(.*)" "$modelStatsTxt" | awk '{ print $4, $5 }')
statsEvalCount=$(grep -oE "^eval count:[[:space:]]+(.*)" "$modelStatsTxt" | awk '{ print $3, $4 }')
statsEvalDuration=$(grep -oE "^eval duration:[[:space:]]+(.*)" "$modelStatsTxt" | awk '{ print $NF }')
statsEvalRate=$(grep -oE "^eval rate:[[:space:]]+(.*)" "$modelStatsTxt" | awk '{ print $3, $4 }')
addedImages=$(grep -oE "Added image '(.*)'" "$modelStatsTxt" | awk '{ print $NF }' | sed "s/'//g")
if [ -n "$addedImages" ]; then
for image in ${addedImages}; do
if ! [ -f "$outputDirectory"/"$(basename $image)" ]; then
echo "Copying image: $image"
cp $image $outputDirectory
fi
done
fi
responseWords=$(wc -w < "$modelOutputTxt" | awk '{print $1}')
responseBytes=$(wc -c < "$modelOutputTxt" | awk '{print $1}')
}
setOllamaStats() {
ollamaVersion=$(ollama -v | awk '{print $4}')
# ps columns: 1:NAME, 2:ID, 3:SIZE_NUM 4:SIZE_GB, 5:PROCESSOR_% 6:PROCESS_TYPE, 7:CONTEXT, 8:UNTIL
ollamaPs=$(ollama ps | awk '{print $1, $2, $3, $4, $5, $6, $7}' | sed '1d') # Get columns from ollama ps output, skipping the header
ollamaModel=$(echo "$ollamaPs" | awk '{print $1}') # Get the model name
ollamaSize=$(echo "$ollamaPs" | awk '{print $3, $4}') # Get the model size
ollamaProcessor=$(echo "$ollamaPs" | awk '{print $5, $6}') # Get the processor
ollamaContext=$(echo "$ollamaPs" | awk '{print $7}') # Get the context size
}
setSystemStats() {
systemArch=$(uname -m) # Get hardware platform
systemProcessor=$(uname -p) # Get system processor
systemOSName=$(uname -s) # Get system OS name
systemOSVersion=$(uname -r) # Get system OS version
setSystemMemoryStats
}
setSystemMemoryStats() {
systemMemoryUsed="?"
systemMemoryAvail="?"
#echo "OS Type: $OSTYPE"
case "$OSTYPE" in
cygwin|msys)
#echo "OS Type match: cygwin|msys"
if command -v wmic >/dev/null 2>&1; then
local totalMemKB=$(wmic OS get TotalVisibleMemorySize /value 2>/dev/null | grep -E "^TotalVisibleMemorySize=" | cut -d'=' -f2 | tr -d '\r')
local availMemKB=$(wmic OS get FreePhysicalMemory /value 2>/dev/null | grep -E "^FreePhysicalMemory=" | cut -d'=' -f2 | tr -d '\r')
if [ -n "$totalMemKB" ] && [ -n "$availMemKB" ]; then
local usedMemKB=$((totalMemKB - availMemKB))
# Convert KB to human readable format (approximate)
if [ $usedMemKB -gt 1048576 ]; then
systemMemoryUsed="$((usedMemKB / 1048576))G"
elif [ $usedMemKB -gt 1024 ]; then
systemMemoryUsed="$((usedMemKB / 1024))M"
else
systemMemoryUsed="${usedMemKB}K"
fi
if [ $availMemKB -gt 1048576 ]; then
systemMemoryAvail="$((availMemKB / 1048576))G"
elif [ $availMemKB -gt 1024 ]; then
systemMemoryAvail="$((availMemKB / 1024))M"
else
systemMemoryAvail="${availMemKB}K"
fi
fi
fi
;;
darwin*)
#echo "OS Type match: darwin"
top=$(top -l 1 2>/dev/null || echo "")
if [ -n "$top" ]; then
systemMemoryUsed=$(echo "$top" | awk '/PhysMem/ {print $2}' || echo "N/A")
systemMemoryAvail=$(echo "$top" | awk '/PhysMem/ {print $6}' || echo "N/A")
fi
;;
*)
#echo "OS Type match: *"
top=$(top -l 1 2>/dev/null || top -bn1 2>/dev/null || echo "")
if [ -n "$top" ]; then
systemMemoryUsed=$(echo "$top" | awk '/PhysMem/ {print $2}' || echo "N/A")
systemMemoryAvail=$(echo "$top" | awk '/PhysMem/ {print $6}' || echo "N/A")
fi
;;
esac
}
showSystemStats() {
echo ""
echo "System |
"
echo "Ollama proc | $ollamaProcessor |
"
echo "Ollama context | $ollamaContext |
"
echo "Ollama version | $ollamaVersion |
"
echo "Multirun timeout | $TIMEOUT seconds |
"
echo "Sys arch | $systemArch |
"
echo "Sys processor | $systemProcessor |
"
echo "sys memory | $systemMemoryUsed + $systemMemoryAvail |
"
echo "Sys OS | $systemOSName $systemOSVersion |
"
echo "
"
}
createModelInfoTxt() { # Create model info files - for each model, do 'ollama show' and save the results to text file
for model in "${models[@]}"; do
modelInfoTxt="$outputDirectory/$(safeString "$model").info.txt"
echo $(getDateTime) "Creating Model Info Text: $modelInfoTxt"
ollama show "$model" > "$modelInfoTxt"
done
}
setModelInfo() {
modelInfoTxt="$outputDirectory/$(safeString "$model").info.txt"
modelCapabilities=()
modelSystemPrompt=""
modelTemperature=""
section=""
while IFS= read -r line; do # Read the content of the file line by line
line="$(echo -e "${line}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" # Trim leading/trailing whitespace
if [[ -z "$line" ]]; then
section=""
continue; # Skip empty lines
fi
if [[ $line == "Model"* ]]; then
section="Model"
continue
elif [[ $line == "Capabilities"* ]]; then
section="Capabilities"
continue
elif [[ $line == "System"* ]]; then
section="System"
continue
elif [[ $line == "Parameters"* ]]; then
section="Parameters"
continue
elif [[ $line == "License"* ]]; then
section="License"
continue
elif [[ $line == "Projector"* ]]; then
section="Projector"
continue
fi
case $section in
"Model")
if [[ "$line" == "architecture"* ]]; then
modelArchitecture=$(echo "$line" | awk '/architecture/ {print $2}') # Get model architecture
fi
if [[ "$line" == "parameters"* ]]; then
modelParameters=$(echo "$line" | awk '/parameters/ {print $2}') # Get model parameters
fi
if [[ "$line" == "context length"* ]]; then
modelContextLength=$(echo "$line" | awk '/context length/ {print $3}') # Get model context length
fi
if [[ "$line" == "embedding length"* ]]; then
modelEmbeddingLength=$(echo "$line" | awk '/embedding length/ {print $3}') # Get model embedding length
fi
if [[ "$line" == "quantization"* ]]; then
modelQuantization=$(echo "$line" | awk '/quantization/ {print $2}') # Get model quantization
fi
;;
"Capabilities")
modelCapabilities+=("$line")
;;
"System")
modelSystemPrompt+="$line"$'\n'
;;
"Parameters")
if [[ "$line" == "temperature"* ]]; then
modelTemperature=$(echo "$line" | awk '/temperature/ {print $2}') # Get model temperature
fi
;;
esac
done < "$modelInfoTxt"
}
createModelsOverviewHtml() {
# list of models used in current run
modelsIndexHtml="$outputDirectory/models.html"
echo $(getDateTime) "Creating Models Index Page: $modelsIndexHtml"
{
showHeader "$NAME: models"
titleLink="$NAME: $tag: models: $tagDatetime"
echo ""
cat <<- EOF
model |
architecture |
parameters |
context length |
embedding length |
quantization |
temperature |
capabilities |
system prompt |
(raw) |
(index) |
EOF
} > "$modelsIndexHtml"
for model in "${models[@]}"; do
setModelInfo
{
echo ""
echo "$model | "
echo "$modelArchitecture | "
echo "$modelParameters | "
echo "$modelContextLength | "
echo "$modelEmbeddingLength | "
echo "$modelQuantization | "
echo "$modelTemperature | "
echo "$(printf "%s " "${modelCapabilities[@]}") | "
echo "$modelSystemPrompt | "
echo "raw | "
echo "index | "
echo "
"
} >> "$modelsIndexHtml"
done
{
echo "
"
showSortableTablesJavascript
showFooter "$titleLink"
} >> "$modelsIndexHtml"
}
createModelOutputHtml() {
modelHtmlFile="$outputDirectory/$(safeString "$model").html"
echo $(getDateTime) "Creating Model Output Page: $modelHtmlFile"
{
showHeader "$NAME: $model"
titleLink="$NAME: $tag: $model: $tagDatetime"
echo "$titleLink
"
createMenu "$model"
echo ""
showPrompt
showImages
modelThinkingTxt="$outputDirectory/$(safeString "$model").thinking.txt"
if [ -f "$modelThinkingTxt" ]; then
echo "Thinking: $model (raw)
"
textarea "$(cat "$modelThinkingTxt")" 3 15 # 3 padding, max 15 lines
echo "
"
fi
echo "Output: $model (raw)
"
textarea "$(cat "$modelOutputTxt")" 3 25 # 3 padding, max 25 lines
echo "
"
echo ""
echo "Stats (raw) |
"
echo "words | $responseWords |
"
echo "bytes | $responseBytes |
"
echo "total duration | $statsTotalDuration |
"
echo "load duration | $statsLoadDuration |
"
echo "prompt eval count | $statsPromptEvalCount |
"
echo "prompt eval duration | $statsPromptEvalDuration |
"
echo "prompt eval rate | $statsPromptEvalRate |
"
echo "eval count | $statsEvalCount |
"
echo "eval duration | $statsEvalDuration |
"
echo "eval rate | $statsEvalRate |
"
echo "
"
echo ""
echo "Model (raw) |
"
echo "name | $model |
"
echo "architecture | $modelArchitecture |
"
echo "size | $ollamaSize |
"
echo "parameters | $modelParameters |
"
echo "context length | $modelContextLength |
"
echo "embedding length | $modelEmbeddingLength |
"
echo "quantization | $modelQuantization |
"
echo "capabilities | $(printf "%s " "${modelCapabilities[@]}") | "
echo "
"
showSystemStats
showFooter "$titleLink"
} > "$modelHtmlFile"
}
createOutputIndexHtml() {
outputIndexHtml="$outputDirectory/index.html"
echo $(getDateTime) "Creating Output Index Page: $outputIndexHtml"
{
showHeader "$NAME: $tag"
titleLink="$NAME: $tag: $tagDatetime"
echo "$titleLink
"
createMenu "index"
echo ""
showPrompt
echo ""
cat <<- "EOF"
model |
words |
bytes |
total duration |
load duration |
prompt eval count |
prompt eval duration |
prompt eval rate |
eval count |
eval duration |
eval rate |
EOF
} > "$outputIndexHtml"
}
addModelToOutputIndexHtml() {
(
echo ""
echo "$model | "
echo "$responseWords | "
echo "$responseBytes | "
echo "$statsTotalDuration | "
echo "$statsLoadDuration | "
echo "$statsPromptEvalCount | "
echo "$statsPromptEvalDuration | "
echo "$statsPromptEvalRate | "
echo "$statsEvalCount | "
echo "$statsEvalDuration | "
echo "$statsEvalRate | "
echo "
"
) >> "$outputIndexHtml"
}
finishOutputIndexHtml() {
{
echo "
"
echo "
"
showSystemStats
showSortableTablesJavascript
titleLink="$NAME: $tag: $tagDatetime"
showFooter "$titleLink"
} >> "$outputIndexHtml"
imagesHtml=$(showImages)
sed -i '' -e "s##${imagesHtml}#" "$outputIndexHtml"
}
getSortedResultsDirectories() {
# Sort directories by datetime at end of directory name
echo $(ls -d "$resultsDirectory"/* | awk 'match($0, /[0-9]{8}-[0-9]{6}$/) { print $0, substr($0, RSTART, RLENGTH) }' | sort -k2 -r | cut -d' ' -f1)
}
createMainIndexHtml() {
resultsIndexFile="${resultsDirectory}/index.html"
echo $(getDateTime) "Creating Main Index Page: $resultsIndexFile"
{
showHeader "$NAME: results"
titleLink="$NAME"
echo ""
echo "Models Index
"
echo "Runs:
"
for dir in $(getSortedResultsDirectories); do
if [ -d "$dir" ]; then
echo "- ${dir##*/}
"
fi
done
echo "
"
showFooter "$titleLink"
} > $resultsIndexFile
}
createMainModelIndexHtml() {
# create table of contents: list all models used in all run results, and links to every individual model run
modelsFound=()
modelsIndex=()
for dir in $(getSortedResultsDirectories); do # for each item in main results directory
if [ -d "$dir" ]; then # if is a directory
for file in "$dir"/*.html; do # for each *.html file in the directory
if [[ $file != *"/index.html" && $file != *"/models.html" ]]; then # skip index.html and models.html
fileName="${file##*/}"
modelName="${fileName%.html}" # remove .html to get model name
if [[ ! "${modelsFound[@]}" =~ "$modelName" ]]; then
modelsFound+=("$modelName")
fi
modelsIndex+=("$modelName:$dir/$fileName")
fi
done
fi
done
mainModelIndexHtml="$resultsDirectory/models.html"
echo; echo $(getDateTime) "Creating Main Model Index Page: $mainModelIndexHtml"
{
showHeader "$NAME: Model Run Index"
titleLink="$NAME: Model Run Index"
echo ""
echo 'Models: '
for foundModel in "${modelsFound[@]}"; do
echo "$foundModel "
done
echo '
'
echo ""
for foundModel in "${modelsFound[@]}"; do
echo " - $foundModel
"
echo " "
for modelIndex in "${modelsIndex[@]}"; do
modelName=${modelIndex%%:*} # get everything before the :
if [ "$modelName" == "$foundModel" ]; then
run=${modelIndex#*:} # get everything after the :
runLink="${run#$resultsDirectory/}" # remove the results directory from beginning
runName="${runLink%/*}" # remove everything after last slash including the slash
echo " - $runName
"
fi
done
echo '
'
done
echo "
"
echo "top
"
showFooter "$titleLink"
} > "$mainModelIndexHtml"
}
runModelWithTimeout() {
ollama run --verbose "${model}" -- "${prompt}" > "${modelOutputTxt}" 2> "${modelStatsTxt}" &
pid=$!
(
sleep $TIMEOUT
if kill -0 $pid 2>/dev/null; then
echo "[ERROR: Multirun Timeout after ${TIMEOUT} seconds]" > "${modelOutputTxt}"
kill $pid 2>/dev/null
fi
) &
timeout_pid=$!
# Wait for the main process to complete
if wait $pid 2>/dev/null; then
# Main process completed successfully, kill the timeout process
if kill -0 $timeout_pid 2>/dev/null; then
kill $timeout_pid 2>/dev/null
wait $timeout_pid 2>/dev/null # Clean up the timeout process
fi
else
# Main process was killed (likely by timeout), wait for timeout process
wait $timeout_pid 2>/dev/null
fi
}
parseThinkingOutput() {
local modelThinkingTxt="$outputDirectory/$(safeString "$model").thinking.txt"
# Check for either tags or Thinking... patterns
if grep -q -E "(|Thinking\.\.\.)" "$modelOutputTxt"; then
#echo "Found thinking content in $modelOutputTxt, extracting..."
# Read the entire file content
local content=$(cat "$modelOutputTxt")
# Extract thinking content
local thinkingContent=""
thinkingContent+=$(echo "$content" | sed -n '//,/<\/think>/p' | sed '1d;$d')
thinkingContent+=$(echo "$content" | sed -n '/Thinking\.\.\./,/\.\.\.done thinking\./p' | sed '1d;$d')
# Remove thinking content from original
content=$(echo "$content" | sed '//,/<\/think>/d')
content=$(echo "$content" | sed '/Thinking\.\.\./,/\.\.\.done thinking\./d')
echo $(getDateTime) "Creating Thinking Text: $modelThinkingTxt"
echo "$thinkingContent" > "$modelThinkingTxt"
echo $(getDateTime) "Updating Model Output Text: $modelOutputTxt"
echo "$content" > "$modelOutputTxt"
fi
}
export OLLAMA_MAX_LOADED_MODELS=1
parseCommandLine "$@"
echo; echo "$NAME v$VERSION"; echo
setModels
setPrompt
echo; echo $(getDateTime) "Response Timeout: $TIMEOUT"
createOutputDirectory
createMainIndexHtml
savePrompt
createModelInfoTxt
createModelsOverviewHtml
setSystemStats
createOutputIndexHtml
for model in "${models[@]}"; do # Loop through each model and run it with the given prompt
echo; echo $(getDateTime) "Running model: $model"
clearModel "$model"
modelOutputTxt="$outputDirectory/$(safeString "$model").output.txt"
modelStatsTxt="$outputDirectory/$(safeString "$model").stats.txt"
echo $(getDateTime) "Creating Model Output Text: $modelOutputTxt"
echo $(getDateTime) "Creating Model Stats Text: $modelStatsTxt"
runModelWithTimeout
setSystemMemoryStats
setOllamaStats
parseThinkingOutput
setModelInfo
setStats
createModelOutputHtml
addModelToOutputIndexHtml
stopModel "$model"
done
finishOutputIndexHtml
createMainModelIndexHtml
echo; echo $(getDateTime) "Done: $outputDirectory/"