Skip to content

Commit

Permalink
more broken json workarounds
Browse files Browse the repository at this point in the history
  • Loading branch information
@borisbat
borisbat committed Apr 1, 2023
1 parent d19bad2 commit 24c91a3
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 17 deletions.
10 changes: 9 additions & 1 deletion daslib/json.das
View file
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,7 @@ def try_fixing_broken_json ( var bad:string )
// skipping whitespace
var nested = i
var j = i + 1
var any_white_space = j < lstr && is_white_space(int(str[j]))
while j < lstr && is_white_space(int(str[j]))
j ++
if j >= lstr // if eof we done, we fix close quote and we are done
Expand All @@ -414,7 +415,14 @@ def try_fixing_broken_json ( var bad:string )
writer |> write_char(' ')
i = j + 1
goto label 0
// ok, its not a string concatination, or a valid character. we assume its nested quotes and we replace with \"
// ok, its not a string concatination, or a valid character
// if it was a whitespace after the quote, we assume its missing ','
if any_white_space
writer |> write_char('"')
writer |> write_char(',')
i = nested + 1
continue
// . we assume its nested quotes and we replace with \"
writer |> write("\\\"")
i = nested + 1
while i < lstr && str[i]!='"'u8
Expand Down
18 changes: 9 additions & 9 deletions modules/dasOpenAI/openai/openai.das
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ def public openai_retrive_model ( model_id:string )
if resp.status_code == HTTP_CURL_FAILED
LAST_ERROR = resp.error
elif resp.status_code != 200
LAST_ERROR = "HTTPS GET failed with status {resp.status_code}"
LAST_ERROR = "HTTPS GET failed with status {resp.status_code}\n{resp.text}"
else
var JV = read_json(resp.text,LAST_ERROR)
if is_valid_response(JV)
Expand Down Expand Up @@ -219,7 +219,7 @@ def public openai_create_completion ( completion:Completion )
if resp.status_code == HTTP_CURL_FAILED
LAST_ERROR = resp.error
elif resp.status_code != 200
LAST_ERROR = "HTTPS POST failed with status {resp.status_code}"
LAST_ERROR = "HTTPS POST failed with status {resp.status_code}\n{resp.text}"
else
var JV <- read_json(resp.text,LAST_ERROR)
if is_valid_response(JV)
Expand Down Expand Up @@ -297,7 +297,7 @@ def public openai_create_chat_completion ( completion:ChatCompletion )
if resp.status_code == HTTP_CURL_FAILED
LAST_ERROR = resp.error
elif resp.status_code != 200
LAST_ERROR = "HTTPS POST failed with status {resp.status_code}"
LAST_ERROR = "HTTPS POST failed with status {resp.status_code}\n{resp.text}"
else
var JV <- read_json(resp.text,LAST_ERROR)
if is_valid_response(JV)
Expand Down Expand Up @@ -354,7 +354,7 @@ def public openai_create_edit ( completion:Edit )
if resp.status_code == HTTP_CURL_FAILED
LAST_ERROR = resp.error
elif resp.status_code != 200
LAST_ERROR = "HTTPS POST failed with status {resp.status_code}"
LAST_ERROR = "HTTPS POST failed with status {resp.status_code}\n{resp.text}"
else
var JV <- read_json(resp.text,LAST_ERROR)
if is_valid_response(JV)
Expand Down Expand Up @@ -403,7 +403,7 @@ def public openai_create_image ( completion:Image )
if resp.status_code == HTTP_CURL_FAILED
LAST_ERROR = resp.error
elif resp.status_code != 200
LAST_ERROR = "HTTPS POST failed with status {resp.status_code}"
LAST_ERROR = "HTTPS POST failed with status {resp.status_code}\n{resp.text}"
else
var JV <- read_json(resp.text,LAST_ERROR)
if is_valid_image_response(JV)
Expand Down Expand Up @@ -437,7 +437,7 @@ def public openai_create_image_variation ( completion:ImageVariation )
if resp.status_code == HTTP_CURL_FAILED
LAST_ERROR = resp.error
elif resp.status_code != 200
LAST_ERROR = "HTTPS POST failed with status {resp.status_code}"
LAST_ERROR = "HTTPS POST failed with status {resp.status_code}\n{resp.text}"
else
var JV <- read_json(resp.text,LAST_ERROR)
if is_valid_image_response(JV)
Expand Down Expand Up @@ -475,7 +475,7 @@ def public openai_create_image_edit ( completion:ImageEdit )
if resp.status_code == HTTP_CURL_FAILED
LAST_ERROR = resp.error
elif resp.status_code != 200
LAST_ERROR = "HTTPS POST failed with status {resp.status_code}"
LAST_ERROR = "HTTPS POST failed with status {resp.status_code}\n{resp.text}"
else
var JV <- read_json(resp.text,LAST_ERROR)
if is_valid_image_response(JV)
Expand Down Expand Up @@ -515,7 +515,7 @@ def public openai_create_transcription ( completion:Transcription )
if resp.status_code == HTTP_CURL_FAILED
ccr.error = resp.error
elif resp.status_code != 200
ccr.error = "HTTPS POST failed with status {resp.status_code}"
ccr.error = "HTTPS POST failed with status {resp.status_code}\n{resp.text}"
else
ccr.text = resp.text
return <- ccr
Expand Down Expand Up @@ -547,7 +547,7 @@ def public openai_create_translation ( completion:Translation )
if resp.status_code == HTTP_CURL_FAILED
ccr.error = resp.error
elif resp.status_code != 200
ccr.error = "HTTPS POST failed with status {resp.status_code}"
ccr.error = "HTTPS POST failed with status {resp.status_code}\n{resp.text}"
else
ccr.text = resp.text
return <- ccr
26 changes: 19 additions & 7 deletions modules/dasTelegram/examples/godfather_bot.das
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ def generate_completion ( summary_prompt:string )
completion <- openai_create_completion([[Completion()
model = "text-davinci-003",
prompt = summary_prompt,
max_tokens = 512,
max_tokens = 1024,
temperature = 0.
]])
if completion |> is_valid
Expand Down Expand Up @@ -317,7 +317,11 @@ def adjust_completion ( var json : JsonValue? )
obj["draw"] = JV(req)
to_log(LOG_INFO, "adjusting draw to {req}\n")

def generate_chat_reply ( chat_id : string; message:tbotapi::message; var tokens:int& )
def generate_chat_reply ( _chat_id : int64; message:tbotapi::message; var tokens:int& )
let chat_id = "{_chat_id}"
if length(g_chat_messages[chat_id].messages)==1 && empty(g_chat_messages[chat_id].summary)
to_log(LOG_INFO, "generate_chat_reply {chat_id} {message.text} - empty chat, getting summary\n")
g_chat_messages[chat_id].summary = generate_chat_summary(_chat_id, 50, "in english language")
to_log(LOG_INFO, "generate_chat_reply {chat_id} {message.text}\n")
tokens = 0
var messages : array<ChatCompletionMessage>
Expand Down Expand Up @@ -528,7 +532,7 @@ def collect_chat_log ( chat_id:int64; last_n_messages:int )
log |> reverse
return <- log

def generate_chat_summary ( chat_id:int64; last_n_messages:int )
def generate_chat_summary ( chat_id:int64; last_n_messages:int; extp:string)
to_log(LOG_INFO, "Generating chat summary for {chat_id} last {last_n_messages} messages\n\n\n")
var inscope log <- collect_chat_log(chat_id, last_n_messages)
var i = 0
Expand All @@ -541,7 +545,7 @@ def generate_chat_summary ( chat_id:int64; last_n_messages:int )
text = "{text}\n{log[i]}"
i ++
if !empty(text)
let summary_prompt = "Previous section summary:\n{prev_summary}\n---\nText:\n{text}\n---\nSummary of the text above in russian language:"
let summary_prompt = "Previous section summary:\n{prev_summary}\n---\nText:\n{text}\n---\nSummary of the text above {extp}:"
to_log(LOG_INFO, "Generating summary for {length(summary_prompt)} bytes\n{summary_prompt}\n\n\n")
let summary = generate_completion(summary_prompt)
to_log(LOG_INFO, "Summary: {summary}\n\n")
Expand Down Expand Up @@ -599,7 +603,8 @@ def setup_bot_commands
command="status", description="show current chat status";
command="users", description="show users in this chat";
command="chatlog", description="show chat log summary for the last n messages";
command="summary", description="show chat log summary for the last n messages"
command="summary", description="show chat log summary for the last n messages";
command="samara", description="сводка по логу чата за последние n сообщений"
}]
]])
if !telegram_get_last_error() |> empty
Expand Down Expand Up @@ -638,7 +643,14 @@ def process_bot_commands ( message:message )
var max_messages = 100
if !empty(num_substr)
max_messages = int(num_substr)
response = generate_chat_summary(message.chat.id, max_messages)
response = generate_chat_summary(message.chat.id, max_messages, "in english language")
elif message.text |> starts_with("/samara@{g_botConfig.id}")
let len_cmd = length("/samara@{g_botConfig.id}")
let num_substr = message.text |> slice(len_cmd+1)
var max_messages = 100
if !empty(num_substr)
max_messages = int(num_substr)
response = generate_chat_summary(message.chat.id, max_messages, "in russian language")
if !empty(response)
telegram_sendLongMessage([[sendmessage
chat_id = "{message.chat.id}",
Expand Down Expand Up @@ -711,7 +723,7 @@ def main
if is_someone_asking_bot_to_say_something(*r.message)
to_log(LOG_INFO, "someone is asking me to say something\n")
var tokens = 0
let reply = generate_chat_reply("{r.message.chat.id}",*r.message, tokens)
let reply = generate_chat_reply(r.message.chat.id,*r.message, tokens)
if reply.result != "error"
g_chat_messages["{r.message.chat.id}"].total_tokens = tokens
process_chat_reply(*r.message,reply)
Expand Down
14 changes: 14 additions & 0 deletions tests/json/broken.das
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,17 @@ let broken3 = %stringify~
}
%%

let broken4 = %stringify~
{
"result" : "nothing",
"summary" : "User asked for the difference between the words: limit, border, and frontier",
"message" : "Ah, Greg, you come to me on the day of my daughter's wedding and ask me about the difference between limit, border, and frontier. I will give you the definitions of each word so that you can understand the differences: n- A limit is a point beyond which something does not or may not extend or pass. n- A border is the outer part or edge of something. It can also mean the line or frontier area separating political or geographical regions. n- A frontier is a line or zone of contact between two adjoining countries, political systems, or groups, where either one or both are expanding or could expand."
"draw" : null,
"mood" : "🤨",
"code" : null,
}
%%

def test_broken ( t: T?; text:string )
var error : string
var j = read_json(text,error)
Expand All @@ -58,3 +69,6 @@ def broken_json ( t: T? )
test_broken(t,broken2)
t |> run("extra ,") <| @ ( t : T? )
test_broken(t,broken3)
t |> run("missing ,") <| @ ( t : T? )
test_broken(t,broken4)

0 comments on commit 24c91a3

Please sign in to comment.