notebooks/gpt_model_prep_v1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GenAI model tuning prep data\n",
    "\n",
    "This notebook attempts to transform free form data to JSONL format so it can be used by OpenAI model tuning API.\n",
    "\n",
    "20231109 Uses raw.txt, the model code to make processed.jsonl sample.\n",
    "\n",
    "\n",
    "\n",
    "#### input format\n",
    "\n",
    "```\n",
    "sample_code\n",
    "\n",
    "commentary\n",
    "\n",
    "prompt\n",
    "\n",
    "```\n",
    "\n",
    "\n",
    "### output format\n",
    "\n",
    "```\n",
    "{\"messages\": [{\"role\": \"system\", \"content\": \"Marv is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"How far is the Chelsea Cafe from my current location?\"}, {\"role\": \"assistant\", \"content\": \"Around five minutes walk. Are you hungry?\"}]}\n",
    "```\n",
    "\n",
    "Expanded for clarity\n",
    "\n",
    "```\n",
    "{\"messages\": [\n",
    "    {\"role\": \"system\", \n",
    "    \"content\": \"Marv is a factual chatbot that is also sarcastic.\"}, \n",
    "    \n",
    "    {\"role\": \"user\", \n",
    "    \"content\": \"How far is the Chelsea Cafe from my current location?\"}, \n",
    "    \n",
    "    {\"role\": \"assistant\", \n",
    "    \"content\": \"Around five minutes walk. Are you hungry?\"}\n",
    "    ]}\n",
    "```\n",
    "\n",
    "```\n",
    "{\"messages\": [\n",
    "    {\"role\": \"system\", \n",
    "    \"content\": \"context\"}, \n",
    "    \n",
    "    {\"role\": \"user\", \n",
    "    \"content\": prompt}, \n",
    "    \n",
    "    {\"role\": \"assistant\", \n",
    "    \"content\": sample_code, commentary}\n",
    "    ]}\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## for reference here is the openai prompt and result\n",
    "\n",
    "We tried this with the GOOG chat but it didn't work.\n",
    "\n",
    "## prompt\n",
    "\n",
    "```\n",
    "Write code in python to read a text file with this format:\n",
    "\n",
    "```\n",
    "sample_code\n",
    "\n",
    "commentary\n",
    "\n",
    "prompt\n",
    "\n",
    "```\n",
    " \n",
    "and output a jsonl file in this format:\n",
    "\n",
    "{\"messages\": [\n",
    "    {\"role\": \"system\", \n",
    "    \"content\": \"context\"}, \n",
    "    \n",
    "    {\"role\": \"user\", \n",
    "    \"content\": prompt}, \n",
    "    \n",
    "    {\"role\": \"assistant\", \n",
    "    \"content\": sample_code, commentary}\n",
    "    ]}\n",
    "```\n",
    "\n",
    "## result\n",
    "\n",
    "```\n",
    "import json\n",
    "\n",
    "# Function to read the input text file and convert it to JSONL format\n",
    "def convert_text_to_jsonl(input_file, output_file):\n",
    "    messages = []\n",
    "\n",
    "    with open(input_file, 'r') as file:\n",
    "        lines = file.readlines()\n",
    "        if len(lines) != 3:\n",
    "            print(\"Input file does not have the correct format (sample_code, commentary, prompt)\")\n",
    "            return\n",
    "\n",
    "        sample_code = lines[0].strip()\n",
    "        commentary = lines[1].strip()\n",
    "        prompt = lines[2].strip()\n",
    "\n",
    "        messages.append({\"role\": \"system\", \"content\": \"context\"})\n",
    "        messages.append({\"role\": \"user\", \"content\": prompt})\n",
    "        messages.append({\"role\": \"assistant\", \"content\": sample_code, \"commentary\": commentary})\n",
    "\n",
    "    with open(output_file, 'w') as jsonl_file:\n",
    "        for message in messages:\n",
    "            jsonl_file.write(json.dumps(message) + '\\n')\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    input_file = \"input.txt\"  # Replace with your input file's name\n",
    "    output_file = \"output.jsonl\"  # Replace with your desired output file's name\n",
    "\n",
    "    convert_text_to_jsonl(input_file, output_file)\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import pathlib\n",
    "import json\n",
    "\n",
    "import pandas as pd\n",
    "import os\n",
    "import os.path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This can be varied to point to different files.\n",
    "IN_FILE_NAME = \"raw.txt\"\n",
    "OUT_FILE_NAME = \"processed.jsonl\"\n",
    "os.environ['IN_FILE_NAME'] = IN_FILE_NAME\n",
    "os.environ['OUT_FILE_NAME'] = OUT_FILE_NAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current directory is : /workspaces/BALSA/notebooks\n"
     ]
    }
   ],
   "source": [
    "# The current directory will be where this src file is located.\n",
    "# Which is in the notebooks dir of the project\n",
    "dirpath = os.getcwd()\n",
    "print(\"current directory is : \" + dirpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root directory is:  /workspaces/BALSA\n",
      "data directory is:  /workspaces/BALSA/data\n",
      "logs directory is:  /workspaces/BALSA/logs\n"
     ]
    }
   ],
   "source": [
    "# Use pathlib to find the root dir of the git repo\n",
    "root_path = pathlib.PurePath(dirpath).parents[0]\n",
    "data_path = root_path / 'data'\n",
    "logs_path = root_path / 'logs'\n",
    "print(\"root directory is: \", root_path)\n",
    "print(\"data directory is: \",  data_path)\n",
    "print(\"logs directory is: \", logs_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LOGS_DIR_NAME:  /workspaces/BALSA/logs\n",
      "DATA_DIR_NAME:  /workspaces/BALSA/data\n"
     ]
    }
   ],
   "source": [
    "# Create equivalent dir names in the environment\n",
    "# Logs\n",
    "LOGS_DIR_NAME = logs_path.as_posix()\n",
    "print(\"LOGS_DIR_NAME: \", LOGS_DIR_NAME)\n",
    "os.environ['LOGS_DIR_NAME'] = LOGS_DIR_NAME\n",
    "# Data\n",
    "DATA_DIR_NAME = data_path.as_posix()\n",
    "print(\"DATA_DIR_NAME: \", DATA_DIR_NAME)\n",
    "os.environ['DATA_DIR_NAME'] = DATA_DIR_NAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspaces/BALSA/data\n",
      "/workspaces/BALSA/logs\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "# Verify env variables are set\n",
    "echo ${DATA_DIR_NAME}\n",
    "echo ${LOGS_DIR_NAME}\n",
    "echo ${CSV_FILE_NAME}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "SAMPLE_CODE\n",
      "\n",
      "This is sample code line 1\n",
      "This is sample code line 2\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "head -n 5 \"${DATA_DIR_NAME}/${IN_FILE_NAME}\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# routine to build our file via chatGPT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to read the input text file and convert it to JSONL format\n",
    "def convert_text_to_jsonl(input_file, output_file):\n",
    "    lines = []\n",
    "    messages = []\n",
    "    # stub vars\n",
    "    sample_code = []\n",
    "    commentary = []\n",
    "    prompt = []\n",
    "\n",
    "    with open(input_file, 'r') as file:\n",
    "        lines = file.readlines()\n",
    "\n",
    "\n",
    "    # \n",
    "    # find delimiters\n",
    "    #\n",
    "\n",
    "    posn = []\n",
    "    line_nbr = 0\n",
    "    for a_line in lines:\n",
    "        #print(a_line)\n",
    "        if (a_line == \"SAMPLE_CODE\\n\"):\n",
    "            #print(a_line, \" \", line_nbr )\n",
    "            # save that position\n",
    "            posn.append(line_nbr) \n",
    "        if (a_line == \"COMMENTARY\\n\"):\n",
    "            #print(a_line)\n",
    "            posn.append(line_nbr) \n",
    "        if (a_line == \"PROMPT\\n\"):\n",
    "            #print(a_line)\n",
    "            posn.append(line_nbr) \n",
    "        \n",
    "        line_nbr = line_nbr + 1\n",
    "\n",
    "    # Record the last line in file\n",
    "    posn.append(line_nbr) \n",
    "\n",
    "    #print(\"posn: \", posn)\n",
    "\n",
    "    # separate out the parts\n",
    "    code_lines = lines[1+1:posn[1]]\n",
    "    commentary_lines = lines[posn[1]+1:posn[2]]\n",
    "    prompt_lines = lines[posn[2]+1:posn[3]]\n",
    "\n",
    "    # dump the parts\n",
    "    #print(\"===code_lines:===\\n\", code_lines)\n",
    "    #print(\"===commentary_lines:===\\n\", commentary_lines)\n",
    "    #print(\"===prompt_lines:===\\n\", prompt_lines)\n",
    "\n",
    "    a_dict = {}\n",
    "    a_dict['messages'] = []\n",
    "    # a_dict['messages'] = [ \n",
    "    #     {'role':'system',\n",
    "    #      'content': 'you are a helpful assistant'},\n",
    "    #     {'role':'user',\n",
    "    #      'content': prompt_lines},\n",
    "    #     {'role':'assistant',\n",
    "    #      'content': code_lines + ' with commentary: ' + commentary_lines}\n",
    "    # ]\n",
    "\n",
    "    code_commentary_lines = code_lines\n",
    "    code_commentary_lines.append('with commentary: \\n')\n",
    "    # WHY?: using [] or not it appends a generator instead of string.\n",
    "    #code_commentary_lines.append([str(e) for e in commentary_lines])\n",
    "    for a_line in commentary_lines:\n",
    "        code_commentary_lines.append(a_line)\n",
    "    \n",
    "    a_dict['messages'].append({'role':'system',\n",
    "                              'content': 'you are a helpful assistant'})\n",
    "    a_dict['messages'].append({'role':'user',\n",
    "                               'content': prompt_lines})\n",
    "    a_dict['messages'].append({'role':'assistant',\n",
    "                               'content': code_commentary_lines})\n",
    "\n",
    "\n",
    "    #print(a_dict)\n",
    "\n",
    "    with open(output_file, 'w') as jsonl_file:\n",
    "        jsonl_file.write(json.dumps(a_dict) + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "IN_FQPN = data_path /  pathlib.PurePath(IN_FILE_NAME).as_posix()\n",
    "OUT_FQPN = data_path /  pathlib.PurePath(OUT_FILE_NAME).as_posix()\n",
    "\n",
    "convert_text_to_jsonl(IN_FQPN, OUT_FQPN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}