open-source-nlp/05.1.Text_Classification_Examples_in_SparkML_SparkNLP.ipynb

{"cells":[{"cell_type":"markdown","metadata":{"id":"sXatvRX899i0"},"source":["![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)"]},{"cell_type":"markdown","metadata":{"id":"C3UywhO6ykrV"},"source":["[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/open-source-nlp/05.1.Text_Classification_Examples_in_SparkML_SparkNLP.ipynb)"]},{"cell_type":"markdown","metadata":{"id":"034_KMGM9mc_"},"source":["# Text Classification with Spark NLP"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"mSYCCzoJKpIO"},"outputs":[],"source":["! pip install -q pyspark==3.4.1 spark-nlp==5.3.2"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":254},"executionInfo":{"elapsed":158085,"status":"ok","timestamp":1711637936386,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"SDasO3DbKu2Z","outputId":"71d66dee-3c6d-470a-a127-b6881d13a249"},"outputs":[{"output_type":"stream","name":"stdout","text":["Spark NLP version:  5.3.2\n","Apache Spark version:  3.4.1\n"]},{"output_type":"execute_result","data":{"text/plain":["<pyspark.sql.session.SparkSession at 0x7b7512a4e6b0>"],"text/html":["\n","            <div>\n","                <p><b>SparkSession - in-memory</b></p>\n","                \n","        <div>\n","            <p><b>SparkContext</b></p>\n","\n","            <p><a href=\"http://093480c6affd:4040\">Spark UI</a></p>\n","\n","            <dl>\n","              <dt>Version</dt>\n","                <dd><code>v3.4.1</code></dd>\n","              <dt>Master</dt>\n","                <dd><code>local[*]</code></dd>\n","              <dt>AppName</dt>\n","                <dd><code>Spark NLP</code></dd>\n","            </dl>\n","        </div>\n","        \n","            </div>\n","        "]},"metadata":{},"execution_count":2}],"source":["import os\n","import sys\n","\n","import sparknlp\n","\n","from sparknlp.base import *\n","from sparknlp.common import *\n","from sparknlp.annotator import *\n","\n","from pyspark.ml import Pipeline\n","from pyspark.sql import SparkSession\n","\n","import pandas as pd\n","\n","spark = sparknlp.start()\n","\n","print(\"Spark NLP version: \", sparknlp.version())\n","print(\"Apache Spark version: \", spark.version)\n","\n","spark"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"IawvXkLGIPPS"},"outputs":[],"source":["! wget https://raw.githubusercontent.com/JohnSnowLabs/spark-nlp-workshop/master/tutorials/Certification_Trainings/Public/data/news_category_train.csv\n","! wget https://raw.githubusercontent.com/JohnSnowLabs/spark-nlp-workshop/master/tutorials/Certification_Trainings/Public/data/news_category_test.csv"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4356,"status":"ok","timestamp":1711637943192,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"1GkCjlO9IPPh","outputId":"e7970888-902a-429f-95d4-5f30880909a3"},"outputs":[{"output_type":"stream","name":"stdout","text":["+--------+--------------------------------------------------+\n","|category|                                       description|\n","+--------+--------------------------------------------------+\n","|Business| Short sellers, Wall Street's dwindling band of...|\n","|Business| Private investment firm Carlyle Group, which h...|\n","|Business| Soaring crude prices plus worries about the ec...|\n","|Business| Authorities have halted oil export flows from ...|\n","|Business| Tearaway world oil prices, toppling records an...|\n","|Business| Stocks ended slightly higher on Friday but sta...|\n","|Business| Assets of the nation's retail money market mut...|\n","|Business| Retail sales bounced back a bit in July, and n...|\n","|Business|\" After earning a PH.D. in Sociology, Danny Baz...|\n","|Business| Short sellers, Wall Street's dwindling  band o...|\n","|Business| Soaring crude prices plus worries  about the e...|\n","|Business| OPEC can do nothing to douse scorching  oil pr...|\n","|Business| Non OPEC oil exporters should consider  increa...|\n","|Business| WASHINGTON/NEW YORK (Reuters) - The auction fo...|\n","|Business| The dollar tumbled broadly on Friday  after da...|\n","|Business|If you think you may need to help your elderly ...|\n","|Business|The purchasing power of kids is a big part of w...|\n","|Business|There is little cause for celebration in the st...|\n","|Business|The US trade deficit has exploded 19 to a recor...|\n","|Business|Oil giant Shell could be bracing itself for a t...|\n","+--------+--------------------------------------------------+\n","only showing top 20 rows\n","\n"]}],"source":["# newsDF = spark.read.parquet(\"data/news_category.parquet\") >> if it is a parquet\n","\n","newsDF = spark.read \\\n","      .option(\"header\", True) \\\n","      .csv(\"news_category_train.csv\")\n","\n","newsDF.show(truncate=50)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":7,"status":"ok","timestamp":1711637943192,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"hjkkY5FTIPPp","outputId":"fa049e86-5fa4-4a5c-fcc4-3f7f106d86bb"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["[Row(category='Business', description=\" Short sellers, Wall Street's dwindling band of ultra cynics, are seeing green again.\"),\n"," Row(category='Business', description=' Private investment firm Carlyle Group, which has a reputation for making well timed and occasionally controversial plays in the defense industry, has quietly placed its bets on another part of the market.')]"]},"metadata":{},"execution_count":5}],"source":["newsDF.take(2)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1903,"status":"ok","timestamp":1711637945092,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"Xsifu6-cIPPs","outputId":"af4ef07f-e363-4bba-cbc6-c98c42ec9d23"},"outputs":[{"output_type":"stream","name":"stdout","text":["+--------+-----+\n","|category|count|\n","+--------+-----+\n","|   World|30000|\n","|Sci/Tech|30000|\n","|  Sports|30000|\n","|Business|30000|\n","+--------+-----+\n","\n"]}],"source":["from pyspark.sql.functions import col\n","\n","newsDF.groupBy(\"category\") \\\n","    .count() \\\n","    .orderBy(col(\"count\").desc()) \\\n","    .show()"]},{"cell_type":"markdown","metadata":{"id":"iHn5crTvIPPv"},"source":["## Building Classification Pipeline"]},{"cell_type":"markdown","metadata":{"id":"G8zK3RByIPPw"},"source":["### LogReg with CountVectorizer"]},{"cell_type":"markdown","metadata":{"id":"AM7esk5yIPPx"},"source":["Tokenizer: Tokenization\n","\n","stopwordsRemover: Remove Stop Words\n","\n","countVectors: Count vectors (“document-term vectors”)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"qbrPTtZpIPPx"},"outputs":[],"source":["from pyspark.ml.feature import CountVectorizer, HashingTF, IDF, OneHotEncoder, StringIndexer, VectorAssembler, SQLTransformer"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":37808,"status":"ok","timestamp":1711637982898,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"dL-SK95DIPP1","outputId":"1abf27f5-3ffd-4a08-e128-ca7a480380e4"},"outputs":[{"output_type":"stream","name":"stdout","text":["CPU times: user 314 ms, sys: 52.1 ms, total: 366 ms\n","Wall time: 37.7 s\n"]},{"output_type":"execute_result","data":{"text/plain":["120000"]},"metadata":{},"execution_count":8}],"source":["%%time\n","\n","document_assembler = DocumentAssembler() \\\n","    .setInputCol(\"description\") \\\n","    .setOutputCol(\"document\")\n","\n","tokenizer = Tokenizer() \\\n","    .setInputCols([\"document\"]) \\\n","    .setOutputCol(\"token\")\n","\n","normalizer = Normalizer() \\\n","    .setInputCols([\"token\"]) \\\n","    .setOutputCol(\"normalized\")\n","\n","stopwords_cleaner = StopWordsCleaner()\\\n","    .setInputCols(\"normalized\")\\\n","    .setOutputCol(\"cleanTokens\")\\\n","    .setCaseSensitive(False)\n","\n","stemmer = Stemmer() \\\n","    .setInputCols([\"cleanTokens\"]) \\\n","    .setOutputCol(\"stem\")\n","\n","finisher = Finisher() \\\n","    .setInputCols([\"stem\"]) \\\n","    .setOutputCols([\"token_features\"]) \\\n","    .setOutputAsArray(True) \\\n","    .setCleanAnnotations(False)\n","\n","countVectors = CountVectorizer(inputCol=\"token_features\", outputCol=\"features\", vocabSize=10000, minDF=5)\n","\n","label_stringIdx = StringIndexer(inputCol = \"category\", outputCol = \"label\")\n","\n","nlp_pipeline = Pipeline(\n","    stages=[\n","        document_assembler,\n","        tokenizer,\n","        normalizer,\n","        stopwords_cleaner,\n","        stemmer,\n","        finisher,\n","        countVectors,\n","        label_stringIdx\n","])\n","\n","nlp_model = nlp_pipeline.fit(newsDF)\n","\n","processed = nlp_model.transform(newsDF)\n","\n","processed.count()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":15,"status":"ok","timestamp":1711637982899,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"YEtzQbNwIPP4","outputId":"e6fce137-c448-4eba-9495-d30f54b5ba58"},"outputs":[{"output_type":"stream","name":"stdout","text":["+--------------------------------------------------+--------------------------------------------------+\n","|                                       description|                                    token_features|\n","+--------------------------------------------------+--------------------------------------------------+\n","| Short sellers, Wall Street's dwindling band of...|[short, seller, wall, street, dwindl, band, ult...|\n","| Private investment firm Carlyle Group, which h...|[privat, invest, firm, carlyl, group, reput, ma...|\n","| Soaring crude prices plus worries about the ec...|[soar, crude, price, plu, worri, economi, outlo...|\n","| Authorities have halted oil export flows from ...|[author, halt, oil, export, flow, main, pipelin...|\n","| Tearaway world oil prices, toppling records an...|[tearawai, world, oil, price, toppl, record, st...|\n","| Stocks ended slightly higher on Friday but sta...|[stock, end, slightli, higher, fridai, staye, n...|\n","| Assets of the nation's retail money market mut...|[asset, nation, retail, monei, market, mutual, ...|\n","| Retail sales bounced back a bit in July, and n...|[retail, sale, bounc, back, bit, juli, new, cla...|\n","|\" After earning a PH.D. in Sociology, Danny Baz...|[earn, phd, sociologi, danni, bazil, rilei, sta...|\n","| Short sellers, Wall Street's dwindling  band o...|[short, seller, wall, street, dwindl, band, ult...|\n","| Soaring crude prices plus worries  about the e...|[soar, crude, price, plu, worri, economi, outlo...|\n","| OPEC can do nothing to douse scorching  oil pr...|[opec, noth, dous, scorch, oil, price, market, ...|\n","| Non OPEC oil exporters should consider  increa...|[non, opec, oil, export, consid, increas, outpu...|\n","| WASHINGTON/NEW YORK (Reuters) - The auction fo...|[washingtonnew, york, reuter, auction, googl, i...|\n","| The dollar tumbled broadly on Friday  after da...|[dollar, tumbl, broadli, fridai, data, showe, r...|\n","|If you think you may need to help your elderly ...|[think, mai, ne, help, elderli, rel, financ, do...|\n","|The purchasing power of kids is a big part of w...|[purchas, power, kid, big, part, backtoschool, ...|\n","|There is little cause for celebration in the st...|[littl, caus, celebr, stock, market, dai, inves...|\n","|The US trade deficit has exploded 19 to a recor...|[u, trade, deficit, explod, record, bn, oil, co...|\n","|Oil giant Shell could be bracing itself for a t...|[oil, giant, shell, brace, takeov, attempt, pos...|\n","+--------------------------------------------------+--------------------------------------------------+\n","only showing top 20 rows\n","\n"]}],"source":["processed.select('description','token_features').show(truncate=50)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":13,"status":"ok","timestamp":1711637982899,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"Mq7tQ_vsIPP7","outputId":"88c493af-ae98-4c63-a562-8dcb851da4ba"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["[Row(token_features=['short', 'seller', 'wall', 'street', 'dwindl', 'band', 'ultra', 'cynic', 'see', 'green']),\n"," Row(token_features=['privat', 'invest', 'firm', 'carlyl', 'group', 'reput', 'make', 'well', 'time', 'occasion', 'controversi', 'plai', 'defens', 'industri', 'quietli', 'place', 'bet', 'anoth', 'part', 'market'])]"]},"metadata":{},"execution_count":10}],"source":["processed.select('token_features').take(2)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":632,"status":"ok","timestamp":1711637983520,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"5R98lqSgIPP_","outputId":"e92fdc52-7ec3-4920-aedf-8583c1014224"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["[Row(features=SparseVector(10000, {241: 1.0, 384: 1.0, 467: 1.0, 745: 1.0, 838: 1.0, 2227: 1.0, 3678: 1.0, 6131: 1.0, 6250: 1.0})),\n"," Row(features=SparseVector(10000, {26: 1.0, 38: 1.0, 46: 1.0, 68: 1.0, 117: 1.0, 155: 1.0, 182: 1.0, 197: 1.0, 246: 1.0, 304: 1.0, 320: 1.0, 407: 1.0, 428: 1.0, 621: 1.0, 867: 1.0, 2363: 1.0, 2829: 1.0, 2859: 1.0, 6868: 1.0}))]"]},"metadata":{},"execution_count":11}],"source":["processed.select('features').take(2)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":10,"status":"ok","timestamp":1711637983520,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"xfnlWafuIPQC","outputId":"840877dd-57f4-4eb0-efe5-3eab40ef4b31"},"outputs":[{"output_type":"stream","name":"stdout","text":["+--------------------+--------------------+-----+\n","|         description|            features|label|\n","+--------------------+--------------------+-----+\n","| Short sellers, W...|(10000,[241,384,4...|  0.0|\n","| Private investme...|(10000,[26,38,46,...|  0.0|\n","| Soaring crude pr...|(10000,[15,28,46,...|  0.0|\n","| Authorities have...|(10000,[0,32,35,4...|  0.0|\n","| Tearaway world o...|(10000,[1,2,11,28...|  0.0|\n","| Stocks ended sli...|(10000,[3,13,14,2...|  0.0|\n","| Assets of the na...|(10000,[0,4,10,15...|  0.0|\n","| Retail sales bou...|(10000,[0,1,10,15...|  0.0|\n","|\" After earning a...|(10000,[98,99,125...|  0.0|\n","| Short sellers, W...|(10000,[241,384,4...|  0.0|\n","| Soaring crude pr...|(10000,[15,28,46,...|  0.0|\n","| OPEC can do noth...|(10000,[0,24,28,2...|  0.0|\n","| Non OPEC oil exp...|(10000,[0,21,28,3...|  0.0|\n","| WASHINGTON/NEW Y...|(10000,[2,4,13,14...|  0.0|\n","| The dollar tumbl...|(10000,[2,14,72,1...|  0.0|\n","|If you think you ...|(10000,[74,76,143...|  0.0|\n","|The purchasing po...|(10000,[46,54,167...|  0.0|\n","|There is little c...|(10000,[29,46,75,...|  0.0|\n","|The US trade defi...|(10000,[2,32,72,1...|  0.0|\n","|Oil giant Shell c...|(10000,[16,32,103...|  0.0|\n","+--------------------+--------------------+-----+\n","only showing top 20 rows\n","\n"]}],"source":["processed.select('description','features','label').show()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":62434,"status":"ok","timestamp":1711638045949,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"LQwMyjUdIPQF","outputId":"9eff12a1-c43d-483f-9a42-42ed753bb655"},"outputs":[{"output_type":"stream","name":"stdout","text":["Training Dataset Count: 83967\n","Test Dataset Count: 36033\n"]}],"source":["# set seed for reproducibility\n","(trainingData, testData) = processed.randomSplit([0.7, 0.3], seed = 100)\n","print(\"Training Dataset Count: \" + str(trainingData.count()))\n","print(\"Test Dataset Count: \" + str(testData.count()))"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":17,"status":"ok","timestamp":1711638045950,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"cq30cb17IPQH","outputId":"15216273-d354-4f17-ddd4-5c4e72f9b9e7"},"outputs":[{"output_type":"stream","name":"stdout","text":["root\n"," |-- category: string (nullable = true)\n"," |-- description: string (nullable = true)\n"," |-- document: array (nullable = true)\n"," |    |-- element: struct (containsNull = true)\n"," |    |    |-- annotatorType: string (nullable = true)\n"," |    |    |-- begin: integer (nullable = false)\n"," |    |    |-- end: integer (nullable = false)\n"," |    |    |-- result: string (nullable = true)\n"," |    |    |-- metadata: map (nullable = true)\n"," |    |    |    |-- key: string\n"," |    |    |    |-- value: string (valueContainsNull = true)\n"," |    |    |-- embeddings: array (nullable = true)\n"," |    |    |    |-- element: float (containsNull = false)\n"," |-- token: array (nullable = true)\n"," |    |-- element: struct (containsNull = true)\n"," |    |    |-- annotatorType: string (nullable = true)\n"," |    |    |-- begin: integer (nullable = false)\n"," |    |    |-- end: integer (nullable = false)\n"," |    |    |-- result: string (nullable = true)\n"," |    |    |-- metadata: map (nullable = true)\n"," |    |    |    |-- key: string\n"," |    |    |    |-- value: string (valueContainsNull = true)\n"," |    |    |-- embeddings: array (nullable = true)\n"," |    |    |    |-- element: float (containsNull = false)\n"," |-- normalized: array (nullable = true)\n"," |    |-- element: struct (containsNull = true)\n"," |    |    |-- annotatorType: string (nullable = true)\n"," |    |    |-- begin: integer (nullable = false)\n"," |    |    |-- end: integer (nullable = false)\n"," |    |    |-- result: string (nullable = true)\n"," |    |    |-- metadata: map (nullable = true)\n"," |    |    |    |-- key: string\n"," |    |    |    |-- value: string (valueContainsNull = true)\n"," |    |    |-- embeddings: array (nullable = true)\n"," |    |    |    |-- element: float (containsNull = false)\n"," |-- cleanTokens: array (nullable = true)\n"," |    |-- element: struct (containsNull = true)\n"," |    |    |-- annotatorType: string (nullable = true)\n"," |    |    |-- begin: integer (nullable = false)\n"," |    |    |-- end: integer (nullable = false)\n"," |    |    |-- result: string (nullable = true)\n"," |    |    |-- metadata: map (nullable = true)\n"," |    |    |    |-- key: string\n"," |    |    |    |-- value: string (valueContainsNull = true)\n"," |    |    |-- embeddings: array (nullable = true)\n"," |    |    |    |-- element: float (containsNull = false)\n"," |-- stem: array (nullable = true)\n"," |    |-- element: struct (containsNull = true)\n"," |    |    |-- annotatorType: string (nullable = true)\n"," |    |    |-- begin: integer (nullable = false)\n"," |    |    |-- end: integer (nullable = false)\n"," |    |    |-- result: string (nullable = true)\n"," |    |    |-- metadata: map (nullable = true)\n"," |    |    |    |-- key: string\n"," |    |    |    |-- value: string (valueContainsNull = true)\n"," |    |    |-- embeddings: array (nullable = true)\n"," |    |    |    |-- element: float (containsNull = false)\n"," |-- token_features: array (nullable = true)\n"," |    |-- element: string (containsNull = true)\n"," |-- features: vector (nullable = true)\n"," |-- label: double (nullable = false)\n","\n"]}],"source":["trainingData.printSchema()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":105406,"status":"ok","timestamp":1711638151353,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"L1LRyPzWIPQK","outputId":"930cd21e-3185-46a4-9cf2-13d07bad419a"},"outputs":[{"output_type":"stream","name":"stdout","text":["+------------------------------+--------+------------------------------+-----+----------+\n","|                   description|category|                   probability|label|prediction|\n","+------------------------------+--------+------------------------------+-----+----------+\n","|\" The major stock indexes e...|Business|[0.999091873162891,2.262691...|  0.0|       0.0|\n","|\" U.S. stocks ended lower o...|Business|[0.9959067648492996,0.00142...|  0.0|       0.0|\n","| Consumer prices rose by a ...|   World|[0.995775953350137,0.001368...|  3.0|       0.0|\n","| Disappointing retail sales...|   World|[0.9954731744653694,0.00257...|  3.0|       0.0|\n","| Why do mutual funds charge...|Business|[0.9954269201581861,0.00325...|  0.0|       0.0|\n","|Banknorth Group's largest s...|Business|[0.9953602482231324,0.00227...|  0.0|       0.0|\n","|The absolute price of oil h...|Sci/Tech|[0.994984659975905,1.901799...|  1.0|       0.0|\n","|\" Sky high oil prices are l...|Business|[0.9947590437405693,0.00250...|  0.0|       0.0|\n","|\" U.S. stocks were little c...|Business|[0.994181120511856,0.002592...|  0.0|       0.0|\n","|Putnam Investments will pay...|Business|[0.9938881586497762,0.00276...|  0.0|       0.0|\n","+------------------------------+--------+------------------------------+-----+----------+\n","only showing top 10 rows\n","\n"]}],"source":["from pyspark.ml.classification import LogisticRegression\n","\n","lr = LogisticRegression(maxIter=10, regParam=0.3, elasticNetParam=0)\n","\n","lrModel = lr.fit(trainingData)\n","\n","predictions = lrModel.transform(testData)\n","\n","predictions.filter(predictions['prediction'] == 0) \\\n","    .select(\"description\",\"category\",\"probability\",\"label\",\"prediction\") \\\n","    .orderBy(\"probability\", ascending=False) \\\n","    .show(n = 10, truncate = 30)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":33385,"status":"ok","timestamp":1711638184735,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"gPRKc6_1IPQO","outputId":"0b3bdf90-0a01-4036-a934-12bfde90c6d9"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.8989373633709627"]},"metadata":{},"execution_count":16}],"source":["from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n","\n","evaluator = MulticlassClassificationEvaluator(predictionCol=\"prediction\")\n","\n","evaluator.evaluate(predictions)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"7rOj1_VXIPQQ"},"outputs":[],"source":["from sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n","y_true = predictions.select(\"label\")\n","y_true = y_true.toPandas()\n","\n","y_pred = predictions.select(\"prediction\")\n","y_pred = y_pred.toPandas()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":15,"status":"ok","timestamp":1711638251168,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"XEgXvy9KIPQT","outputId":"b5422e8a-f954-4cc1-8513-e14ad64f6bf7"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["1.0    9222\n","2.0    9134\n","0.0    9105\n","3.0    8572\n","Name: prediction, dtype: int64"]},"metadata":{},"execution_count":18}],"source":["y_pred.prediction.value_counts()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":14,"status":"ok","timestamp":1711638251168,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"0AHRzWCZIPQW","outputId":"c6803476-9948-47be-ed1b-d91244f620a7"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([[7874,  845,   84,  328],\n","       [ 716, 7992,   93,  290],\n","       [  48,   90, 8660,   82],\n","       [ 467,  295,  297, 7872]])"]},"metadata":{},"execution_count":19}],"source":["cnf_matrix = confusion_matrix(list(y_true.label.astype(int)), list(y_pred.prediction.astype(int)))\n","cnf_matrix"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":12,"status":"ok","timestamp":1711638251168,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"cTNup21PIPQZ","outputId":"8e6d5629-79d8-4062-aa8b-67323edbffef"},"outputs":[{"output_type":"stream","name":"stdout","text":["              precision    recall  f1-score   support\n","\n","         0.0       0.86      0.86      0.86      9131\n","         1.0       0.87      0.88      0.87      9091\n","         2.0       0.95      0.98      0.96      8880\n","         3.0       0.92      0.88      0.90      8931\n","\n","    accuracy                           0.90     36033\n","   macro avg       0.90      0.90      0.90     36033\n","weighted avg       0.90      0.90      0.90     36033\n","\n","0.8991202508811368\n"]}],"source":["print(classification_report(y_true.label, y_pred.prediction))\n","print(accuracy_score(y_true.label, y_pred.prediction))"]},{"cell_type":"markdown","metadata":{"id":"PSSB-ZHJIPQb"},"source":["### LogReg with TFIDF"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":34257,"status":"ok","timestamp":1711638285414,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"mmWR-GnOIPQc","outputId":"21d52795-1ca9-4def-9f08-7cdd50c3604e"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["120000"]},"metadata":{},"execution_count":21}],"source":["from pyspark.ml.feature import HashingTF, IDF\n","\n","hashingTF = HashingTF(inputCol=\"token_features\", outputCol=\"rawFeatures\", numFeatures=10000)\n","\n","idf = IDF(inputCol=\"rawFeatures\", outputCol=\"features\", minDocFreq=5) #minDocFreq: remove sparse terms\n","\n","nlp_pipeline_tf = Pipeline(\n","    stages=[\n","        document_assembler,\n","        tokenizer,\n","        normalizer,\n","        stopwords_cleaner,\n","        stemmer,\n","        finisher,\n","        hashingTF,\n","        idf,\n","        label_stringIdx\n","])\n","\n","nlp_model_tf = nlp_pipeline_tf.fit(newsDF)\n","\n","processed_tf = nlp_model_tf.transform(newsDF)\n","\n","processed_tf.count()\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":11,"status":"ok","timestamp":1711638285414,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"4RHs0RjOIPQe","outputId":"91734311-3461-4038-d774-944c77ebb524"},"outputs":[{"output_type":"stream","name":"stdout","text":["+--------------------+--------------------+-----+\n","|         description|            features|label|\n","+--------------------+--------------------+-----+\n","| Short sellers, W...|(10000,[551,621,6...|  0.0|\n","| Private investme...|(10000,[157,831,9...|  0.0|\n","| Soaring crude pr...|(10000,[793,1738,...|  0.0|\n","| Authorities have...|(10000,[1548,1611...|  0.0|\n","| Tearaway world o...|(10000,[323,585,1...|  0.0|\n","| Stocks ended sli...|(10000,[453,609,6...|  0.0|\n","| Assets of the na...|(10000,[258,444,1...|  0.0|\n","| Retail sales bou...|(10000,[14,585,19...|  0.0|\n","|\" After earning a...|(10000,[114,796,1...|  0.0|\n","| Short sellers, W...|(10000,[551,621,6...|  0.0|\n","| Soaring crude pr...|(10000,[793,1738,...|  0.0|\n","| OPEC can do noth...|(10000,[298,616,9...|  0.0|\n","| Non OPEC oil exp...|(10000,[616,1063,...|  0.0|\n","| WASHINGTON/NEW Y...|(10000,[360,832,1...|  0.0|\n","| The dollar tumbl...|(10000,[419,949,1...|  0.0|\n","|If you think you ...|(10000,[1041,2059...|  0.0|\n","|The purchasing po...|(10000,[901,2198,...|  0.0|\n","|There is little c...|(10000,[320,515,5...|  0.0|\n","|The US trade defi...|(10000,[419,444,1...|  0.0|\n","|Oil giant Shell c...|(10000,[1336,1998...|  0.0|\n","+--------------------+--------------------+-----+\n","only showing top 20 rows\n","\n"]}],"source":["# set seed for reproducibility\n","processed_tf.select('description','features','label').show()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":64676,"status":"ok","timestamp":1711638350080,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"UHewYwjbIPQg","outputId":"fe8036bb-dc5c-45c2-84f8-5f945729d365"},"outputs":[{"output_type":"stream","name":"stdout","text":["Training Dataset Count: 83967\n","Test Dataset Count: 36033\n"]}],"source":["(trainingData, testData) = processed_tf.randomSplit([0.7, 0.3], seed = 100)\n","print(\"Training Dataset Count: \" + str(trainingData.count()))\n","print(\"Test Dataset Count: \" + str(testData.count()))"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":103863,"status":"ok","timestamp":1711638453934,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"HcCnp1PrIPQj","outputId":"3f3cbfdb-b539-4394-b865-3d4aeb452673"},"outputs":[{"output_type":"stream","name":"stdout","text":["+------------------------------+--------+------------------------------+-----+----------+\n","|                   description|category|                   probability|label|prediction|\n","+------------------------------+--------+------------------------------+-----+----------+\n","|\" The major stock indexes e...|Business|[0.9985997794719548,5.73025...|  0.0|       0.0|\n","|The board overseeing MFS In...|Business|[0.9969443330650403,0.00241...|  0.0|       0.0|\n","|Former Enron Corp. executiv...|Business|[0.9964788037536277,0.00133...|  0.0|       0.0|\n","| Disappointing retail sales...|   World|[0.9961471989168428,0.00187...|  3.0|       0.0|\n","|\" U.S. stocks ended lower o...|Business|[0.9950907314585086,0.00274...|  0.0|       0.0|\n","|\" Mid priced clothing retai...|Business|[0.9938272821165663,0.00197...|  0.0|       0.0|\n","| Consumer prices rose by a ...|   World|[0.993651814252647,0.002007...|  3.0|       0.0|\n","| Oil prices reached a new h...|   World|[0.9930688447636813,7.14155...|  3.0|       0.0|\n","|\" Sears, Roebuck   Co. &lt;...|Business|[0.9925915990802867,0.00281...|  0.0|       0.0|\n","| This year's surge in energ...|   World|[0.9923686240509163,0.00527...|  3.0|       0.0|\n","+------------------------------+--------+------------------------------+-----+----------+\n","only showing top 10 rows\n","\n"]}],"source":["lrModel_tf = lr.fit(trainingData)\n","\n","predictions_tf = lrModel_tf.transform(testData)\n","\n","predictions_tf.select(\"description\",\"category\",\"probability\",\"label\",\"prediction\") \\\n","    .orderBy(\"probability\", ascending=False) \\\n","    .show(n = 10, truncate = 30)\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":67775,"status":"ok","timestamp":1711638521707,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"j8EH6P4-IPQm","outputId":"05191d85-21fa-4674-b9a1-d88f1e970217"},"outputs":[{"output_type":"stream","name":"stdout","text":["              precision    recall  f1-score   support\n","\n","         0.0       0.85      0.85      0.85      9131\n","         1.0       0.85      0.86      0.86      9091\n","         2.0       0.93      0.96      0.95      8880\n","         3.0       0.90      0.87      0.89      8931\n","\n","    accuracy                           0.89     36033\n","   macro avg       0.89      0.89      0.89     36033\n","weighted avg       0.89      0.89      0.89     36033\n","\n","0.8855771098715067\n"]}],"source":["y_true = predictions_tf.select(\"label\")\n","y_true = y_true.toPandas()\n","\n","y_pred = predictions_tf.select(\"prediction\")\n","y_pred = y_pred.toPandas()\n","\n","print(classification_report(y_true.label, y_pred.prediction))\n","print(accuracy_score(y_true.label, y_pred.prediction))"]},{"cell_type":"markdown","metadata":{"id":"6bdwtu_9IPQp"},"source":["### Random Forest with TFIDF"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"ILNZ4-vPIPQp"},"outputs":[],"source":["from pyspark.ml.classification import RandomForestClassifier\n","\n","rf = RandomForestClassifier(labelCol=\"label\", \\\n","                            featuresCol=\"features\", \\\n","                            numTrees = 100, \\\n","                            maxDepth = 4, \\\n","                            maxBins = 32)\n","\n","# Train model with Training Data\n","rfModel = rf.fit(trainingData)\n","predictions_rf = rfModel.transform(testData)\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":34046,"status":"ok","timestamp":1711638693016,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"pY9EQY1tIPQr","outputId":"cea85f62-cf6f-4380-ffb5-51d66e47f15c"},"outputs":[{"output_type":"stream","name":"stdout","text":["+------------------------------+--------+------------------------------+-----+----------+\n","|                   description|category|                   probability|label|prediction|\n","+------------------------------+--------+------------------------------+-----+----------+\n","| Japan's Nikkei average ros...|Business|[0.387888600873362,0.221694...|  0.0|       0.0|\n","|\" Amazon.com Inc. &lt;A HRE...|Business|[0.38669064250683755,0.2342...|  0.0|       0.0|\n","|US stocks opened higher on ...|Business|[0.3847050771462904,0.22817...|  0.0|       0.0|\n","|\" U.S. stocks ended lower o...|Business|[0.3817499960596381,0.23254...|  0.0|       0.0|\n","| Investors bid stocks highe...|   World|[0.3773107429507354,0.24689...|  3.0|       0.0|\n","| Best Buy Company Inc., the...|Business|[0.3762196991895761,0.25226...|  0.0|       0.0|\n","|Nut and snack-food company ...|Business|[0.3746249487873475,0.25013...|  0.0|       0.0|\n","|Crude oil prices hovered ne...|Business|[0.3738391266127697,0.21327...|  0.0|       0.0|\n","|\" Blue chip stocks opened s...|Business|[0.3737234980068536,0.26263...|  0.0|       0.0|\n","| Japan's Nikkei average fel...|Business|[0.3721862729334383,0.24633...|  0.0|       0.0|\n","+------------------------------+--------+------------------------------+-----+----------+\n","only showing top 10 rows\n","\n"]}],"source":["predictions_rf.select(\"description\",\"category\",\"probability\",\"label\",\"prediction\") \\\n","    .orderBy(\"probability\", ascending=False) \\\n","    .show(n = 10, truncate = 30)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":66561,"status":"ok","timestamp":1711638759547,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"7_vXMkvuIPQu","outputId":"1a93aed7-3952-414c-8717-c2bd99dca8d0"},"outputs":[{"output_type":"stream","name":"stdout","text":["              precision    recall  f1-score   support\n","\n","         0.0       0.76      0.67      0.71      9131\n","         1.0       0.81      0.59      0.68      9091\n","         2.0       0.66      0.92      0.77      8880\n","         3.0       0.75      0.76      0.76      8931\n","\n","    accuracy                           0.73     36033\n","   macro avg       0.75      0.73      0.73     36033\n","weighted avg       0.75      0.73      0.73     36033\n","\n","0.7329392501318236\n"]}],"source":["y_true = predictions_rf.select(\"label\")\n","y_true = y_true.toPandas()\n","\n","y_pred = predictions_rf.select(\"prediction\")\n","y_pred = y_pred.toPandas()\n","\n","print(classification_report(y_true.label, y_pred.prediction))\n","print(accuracy_score(y_true.label, y_pred.prediction))"]},{"cell_type":"markdown","metadata":{"id":"z3tdY1_oIPQ0"},"source":["## LogReg with Spark NLP Glove Word Embeddings"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":255770,"status":"ok","timestamp":1711639015301,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"4GVKps1mIPQ0","outputId":"3099cd14-dbb8-458a-aef5-118fdc6ddd6f"},"outputs":[{"output_type":"stream","name":"stdout","text":["glove_100d download started this may take some time.\n","Approximate size to download 145.3 MB\n","[OK!]\n"]},{"output_type":"execute_result","data":{"text/plain":["120000"]},"metadata":{},"execution_count":29}],"source":["document_assembler = DocumentAssembler() \\\n","    .setInputCol(\"description\") \\\n","    .setOutputCol(\"document\")\n","\n","tokenizer = Tokenizer() \\\n","    .setInputCols([\"document\"]) \\\n","    .setOutputCol(\"token\")\n","\n","normalizer = Normalizer() \\\n","    .setInputCols([\"token\"]) \\\n","    .setOutputCol(\"normalized\")\n","\n","stopwords_cleaner = StopWordsCleaner()\\\n","    .setInputCols(\"normalized\")\\\n","    .setOutputCol(\"cleanTokens\")\\\n","    .setCaseSensitive(False)\n","\n","glove_embeddings = WordEmbeddingsModel().pretrained() \\\n","    .setInputCols([\"document\",'cleanTokens'])\\\n","    .setOutputCol(\"embeddings\")\\\n","    .setCaseSensitive(False)\n","\n","embeddingsSentence = SentenceEmbeddings() \\\n","    .setInputCols([\"document\", \"embeddings\"]) \\\n","    .setOutputCol(\"sentence_embeddings\") \\\n","    .setPoolingStrategy(\"AVERAGE\")\n","\n","embeddings_finisher = EmbeddingsFinisher() \\\n","    .setInputCols([\"sentence_embeddings\"]) \\\n","    .setOutputCols([\"finished_sentence_embeddings\"]) \\\n","    .setOutputAsVector(True)\\\n","    .setCleanAnnotations(False)\n","\n","explodeVectors = SQLTransformer(statement=\n","    \"SELECT EXPLODE(finished_sentence_embeddings) AS features, * FROM __THIS__\")\n","\n","label_stringIdx = StringIndexer(inputCol = \"category\", outputCol = \"label\")\n","\n","\n","nlp_pipeline_w2v = Pipeline(\n","    stages=[\n","        document_assembler,\n","        tokenizer,\n","        normalizer,\n","        stopwords_cleaner,\n","        glove_embeddings,\n","        embeddingsSentence,\n","        embeddings_finisher,\n","        explodeVectors,\n","        label_stringIdx\n","])\n","\n","nlp_model_w2v = nlp_pipeline_w2v.fit(newsDF)\n","\n","processed_w2v = nlp_model_w2v.transform(newsDF)\n","\n","processed_w2v.count()\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":12,"status":"ok","timestamp":1711639015301,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"K8kfQldjVdUt","outputId":"0fcf911d-3461-49a2-804a-6e9e55091214"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["['features',\n"," 'category',\n"," 'description',\n"," 'document',\n"," 'token',\n"," 'normalized',\n"," 'cleanTokens',\n"," 'embeddings',\n"," 'sentence_embeddings',\n"," 'finished_sentence_embeddings',\n"," 'label']"]},"metadata":{},"execution_count":30}],"source":["processed_w2v.columns"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":585,"status":"ok","timestamp":1711639015881,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"3iRIsxANVfFM","outputId":"acfbc937-1c47-4839-c70e-a7086d3b07ff"},"outputs":[{"output_type":"stream","name":"stdout","text":["+--------------------+--------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+----------------------------+-----+\n","|            features|category|         description|            document|               token|          normalized|         cleanTokens|          embeddings| sentence_embeddings|finished_sentence_embeddings|label|\n","+--------------------+--------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+----------------------------+-----+\n","|[-0.1556767076253...|Business| Short sellers, W...|[{document, 0, 84...|[{token, 1, 5, Sh...|[{token, 1, 5, Sh...|[{token, 1, 5, Sh...|[{word_embeddings...|[{sentence_embedd...|        [[-0.155676707625...|  0.0|\n","|[-0.0144653050228...|Business| Private investme...|[{document, 0, 20...|[{token, 1, 7, Pr...|[{token, 1, 7, Pr...|[{token, 1, 7, Pr...|[{word_embeddings...|[{sentence_embedd...|        [[-0.014465305022...|  0.0|\n","|[0.10348732769489...|Business| Soaring crude pr...|[{document, 0, 17...|[{token, 1, 7, So...|[{token, 1, 7, So...|[{token, 1, 7, So...|[{word_embeddings...|[{sentence_embedd...|        [[0.1034873276948...|  0.0|\n","|[-0.0355810523033...|Business| Authorities have...|[{document, 0, 18...|[{token, 1, 11, A...|[{token, 1, 11, A...|[{token, 1, 11, A...|[{word_embeddings...|[{sentence_embedd...|        [[-0.035581052303...|  0.0|\n","|[0.00647281948477...|Business| Tearaway world o...|[{document, 0, 15...|[{token, 1, 8, Te...|[{token, 1, 8, Te...|[{token, 1, 8, Te...|[{word_embeddings...|[{sentence_embedd...|        [[0.0064728194847...|  0.0|\n","+--------------------+--------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+----------------------------+-----+\n","only showing top 5 rows\n","\n"]}],"source":["processed_w2v.show(5)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":5,"status":"ok","timestamp":1711639015882,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"RLVVHJEHIPQ3","outputId":"03afb7dc-2b3a-4bf1-a6ff-0f92b5256129"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["[Row(finished_sentence_embeddings=[DenseVector([-0.1557, 0.196, 0.1099, -0.3089, 0.16, 0.1672, -0.4649, -0.1101, -0.053, -0.1551, 0.0327, 0.0772, 0.1494, -0.1865, 0.1155, -0.0597, 0.0234, -0.0451, 0.2361, -0.0089, 0.3358, 0.0444, 0.0088, -0.1453, 0.2289, 0.0914, -0.1665, -0.3726, 0.1892, 0.121, 0.1993, -0.0239, -0.1346, 0.1159, 0.2086, 0.1285, 0.068, 0.1372, 0.3153, -0.1934, 0.0257, -0.226, -0.0984, 0.1139, 0.1413, -0.3743, 0.072, 0.1403, 0.251, -0.3106, 0.1709, -0.0697, -0.0554, 0.5123, -0.1873, -1.7784, 0.0295, 0.1014, 0.9268, 0.2129, -0.1354, 0.5739, -0.0679, 0.461, 0.4216, 0.0225, 0.4456, -0.2462, 0.1411, -0.3258, 0.0025, 0.0114, -0.3895, -0.1106, -0.261, 0.0147, 0.0781, 0.1268, -0.2042, -0.2278, 0.5096, 0.1539, -0.3515, -0.0102, -0.7003, -0.3872, -0.1668, -0.2405, -0.0766, 0.1396, -0.0592, -0.1568, -0.1606, -0.1371, -0.684, -0.2549, -0.1541, 0.1536, 0.2715, 0.3342])])]"]},"metadata":{},"execution_count":32}],"source":["processed_w2v.select('finished_sentence_embeddings').take(1)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"0JRYsk0mIPQ_"},"outputs":[],"source":["# IF SQLTransformer IS NOT USED INSIDE THE PIPELINE, WE CAN EXPLODE OUTSIDE\n","from pyspark.sql.functions import explode\n","\n","# processed_w2v= processed_w2v.withColumn(\"features\", explode(processed_w2v.finished_sentence_embeddings))"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1711639015882,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"ZWp3CA_kIPRB","outputId":"0506e133-ed12-4f83-bc0d-d15716663c57"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["[Row(features=DenseVector([-0.1557, 0.196, 0.1099, -0.3089, 0.16, 0.1672, -0.4649, -0.1101, -0.053, -0.1551, 0.0327, 0.0772, 0.1494, -0.1865, 0.1155, -0.0597, 0.0234, -0.0451, 0.2361, -0.0089, 0.3358, 0.0444, 0.0088, -0.1453, 0.2289, 0.0914, -0.1665, -0.3726, 0.1892, 0.121, 0.1993, -0.0239, -0.1346, 0.1159, 0.2086, 0.1285, 0.068, 0.1372, 0.3153, -0.1934, 0.0257, -0.226, -0.0984, 0.1139, 0.1413, -0.3743, 0.072, 0.1403, 0.251, -0.3106, 0.1709, -0.0697, -0.0554, 0.5123, -0.1873, -1.7784, 0.0295, 0.1014, 0.9268, 0.2129, -0.1354, 0.5739, -0.0679, 0.461, 0.4216, 0.0225, 0.4456, -0.2462, 0.1411, -0.3258, 0.0025, 0.0114, -0.3895, -0.1106, -0.261, 0.0147, 0.0781, 0.1268, -0.2042, -0.2278, 0.5096, 0.1539, -0.3515, -0.0102, -0.7003, -0.3872, -0.1668, -0.2405, -0.0766, 0.1396, -0.0592, -0.1568, -0.1606, -0.1371, -0.684, -0.2549, -0.1541, 0.1536, 0.2715, 0.3342]))]"]},"metadata":{},"execution_count":34}],"source":["processed_w2v.select(\"features\").take(1)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":446,"status":"ok","timestamp":1711639016326,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"yMN2OwXlIPRF","outputId":"7b478507-eaa5-4045-a306-7da0eca6ae61"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["[Row(features=DenseVector([-0.1557, 0.196, 0.1099, -0.3089, 0.16, 0.1672, -0.4649, -0.1101, -0.053, -0.1551, 0.0327, 0.0772, 0.1494, -0.1865, 0.1155, -0.0597, 0.0234, -0.0451, 0.2361, -0.0089, 0.3358, 0.0444, 0.0088, -0.1453, 0.2289, 0.0914, -0.1665, -0.3726, 0.1892, 0.121, 0.1993, -0.0239, -0.1346, 0.1159, 0.2086, 0.1285, 0.068, 0.1372, 0.3153, -0.1934, 0.0257, -0.226, -0.0984, 0.1139, 0.1413, -0.3743, 0.072, 0.1403, 0.251, -0.3106, 0.1709, -0.0697, -0.0554, 0.5123, -0.1873, -1.7784, 0.0295, 0.1014, 0.9268, 0.2129, -0.1354, 0.5739, -0.0679, 0.461, 0.4216, 0.0225, 0.4456, -0.2462, 0.1411, -0.3258, 0.0025, 0.0114, -0.3895, -0.1106, -0.261, 0.0147, 0.0781, 0.1268, -0.2042, -0.2278, 0.5096, 0.1539, -0.3515, -0.0102, -0.7003, -0.3872, -0.1668, -0.2405, -0.0766, 0.1396, -0.0592, -0.1568, -0.1606, -0.1371, -0.684, -0.2549, -0.1541, 0.1536, 0.2715, 0.3342]))]"]},"metadata":{},"execution_count":35}],"source":["processed_w2v.select(\"features\").take(1)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":511,"status":"ok","timestamp":1711639016835,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"ViXxQrhSIPRH","outputId":"ec96682d-9dca-4776-8450-4a17cacd8bfb"},"outputs":[{"output_type":"stream","name":"stdout","text":["+--------------------+--------------------+-----+\n","|         description|            features|label|\n","+--------------------+--------------------+-----+\n","| Short sellers, W...|[-0.1556767076253...|  0.0|\n","| Private investme...|[-0.0144653050228...|  0.0|\n","| Soaring crude pr...|[0.10348732769489...|  0.0|\n","| Authorities have...|[-0.0355810523033...|  0.0|\n","| Tearaway world o...|[0.00647281948477...|  0.0|\n","| Stocks ended sli...|[0.20069395005702...|  0.0|\n","| Assets of the na...|[0.38012433052062...|  0.0|\n","| Retail sales bou...|[0.20352847874164...|  0.0|\n","|\" After earning a...|[0.13536226749420...|  0.0|\n","| Short sellers, W...|[-0.1556767076253...|  0.0|\n","| Soaring crude pr...|[0.10348732769489...|  0.0|\n","| OPEC can do noth...|[0.20307321846485...|  0.0|\n","| Non OPEC oil exp...|[0.09010648727416...|  0.0|\n","| WASHINGTON/NEW Y...|[0.10887209326028...|  0.0|\n","| The dollar tumbl...|[0.05723679438233...|  0.0|\n","|If you think you ...|[0.11463439464569...|  0.0|\n","|The purchasing po...|[0.05890964344143...|  0.0|\n","|There is little c...|[0.11162020266056...|  0.0|\n","|The US trade defi...|[0.09235177189111...|  0.0|\n","|Oil giant Shell c...|[0.01885983347892...|  0.0|\n","+--------------------+--------------------+-----+\n","only showing top 20 rows\n","\n"]}],"source":["processed_w2v.select('description','features','label').show()\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":220152,"status":"ok","timestamp":1711639236986,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"Vx__5MOLIPRK","outputId":"c5d3c2bf-e8f4-4749-d4db-c35668add4ca"},"outputs":[{"output_type":"stream","name":"stdout","text":["Training Dataset Count: 83967\n","Test Dataset Count: 36033\n"]}],"source":["# set seed for reproducibility\n","(trainingData, testData) = processed_w2v.randomSplit([0.7, 0.3], seed = 100)\n","print(\"Training Dataset Count: \" + str(trainingData.count()))\n","print(\"Test Dataset Count: \" + str(testData.count()))"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"FK6x3ljGIPRO"},"outputs":[],"source":["from pyspark.sql.functions import udf\n","\n","@udf(\"long\")\n","def num_nonzeros(v):\n","    return v.numNonzeros()\n","\n","testData = testData.where(num_nonzeros(\"features\") != 0)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"jdbLwEsnIPRR"},"outputs":[],"source":["lrModel_w2v = lr.fit(trainingData)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":115326,"status":"ok","timestamp":1711639575735,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"FoSYCqDjIPRT","outputId":"6bc6015e-1a56-4096-f24d-b9a6b0a140d4"},"outputs":[{"output_type":"stream","name":"stdout","text":["+------------------------------+--------+------------------------------+-----+----------+\n","|                   description|category|                   probability|label|prediction|\n","+------------------------------+--------+------------------------------+-----+----------+\n","|THE stock market is rising,...|Business|[0.9897492276739599,0.00522...|  0.0|       0.0|\n","| Stocks fell on Tuesday as ...|Business|[0.9886070356102349,0.00579...|  0.0|       0.0|\n","|Swiss banking giant UBS AG ...|Business|[0.9885750808549393,0.00622...|  0.0|       0.0|\n","|Tokyo stocks plunged Tuesda...|Business|[0.9843279839213964,0.00453...|  0.0|       0.0|\n","| Financial services company...|Business|[0.9841911568755125,0.00924...|  0.0|       0.0|\n","|Hutchison Whampoa said it w...|Business|[0.9815622819151237,0.01703...|  0.0|       0.0|\n","| Citigroup Inc.  on Thursda...|Business|[0.9814307059494389,0.01481...|  0.0|       0.0|\n","|\" Stocks fell sharply on We...|Business|[0.9806525515660117,0.00859...|  0.0|       0.0|\n","|Newmont Mining Corp. on Wed...|Business|[0.9805074384489912,0.01414...|  0.0|       0.0|\n","|Stocks opened higher today,...|Business|[0.9796643981172639,0.01294...|  0.0|       0.0|\n","+------------------------------+--------+------------------------------+-----+----------+\n","only showing top 10 rows\n","\n"]}],"source":["predictions_w2v = lrModel_w2v.transform(testData)\n","\n","predictions_w2v.select(\"description\",\"category\",\"probability\",\"label\",\"prediction\") \\\n","    .orderBy(\"probability\", ascending=False) \\\n","    .show(n = 10, truncate = 30)\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":221663,"status":"ok","timestamp":1711639797395,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"P2WQ3rZmIPRX","outputId":"ef5bd049-7bb0-46b7-b740-57c46fc42333"},"outputs":[{"output_type":"stream","name":"stdout","text":["              precision    recall  f1-score   support\n","\n","         0.0       0.82      0.82      0.82      8976\n","         1.0       0.83      0.81      0.82      9034\n","         2.0       0.93      0.96      0.94      8972\n","         3.0       0.88      0.87      0.87      9050\n","\n","    accuracy                           0.87     36032\n","   macro avg       0.86      0.87      0.87     36032\n","weighted avg       0.86      0.87      0.87     36032\n","\n","0.8655916962699822\n"]}],"source":["from sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n","import pandas as pd\n","\n","y_true = predictions_w2v.select(\"label\")\n","y_true = y_true.toPandas()\n","\n","y_pred = predictions_w2v.select(\"prediction\")\n","y_pred = y_pred.toPandas()\n","\n","print(classification_report(y_true.label, y_pred.prediction))\n","print(accuracy_score(y_true.label, y_pred.prediction))"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":11,"status":"ok","timestamp":1711639797395,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"idNRXJJiIPRa","outputId":"d5b5102d-03c8-4bed-be02-41deb9c2490e"},"outputs":[{"output_type":"stream","name":"stdout","text":["+--------------------------------------------------+--------------------------------------------------+\n","|                                       description|                                            result|\n","+--------------------------------------------------+--------------------------------------------------+\n","| Short sellers, Wall Street's dwindling band of...|[Short, sellers, Wall, Streets, dwindling, band...|\n","| Private investment firm Carlyle Group, which h...|[Private, investment, firm, Carlyle, Group, rep...|\n","| Soaring crude prices plus worries about the ec...|[Soaring, crude, prices, plus, worries, economy...|\n","| Authorities have halted oil export flows from ...|[Authorities, halted, oil, export, flows, main,...|\n","| Tearaway world oil prices, toppling records an...|[Tearaway, world, oil, prices, toppling, record...|\n","| Stocks ended slightly higher on Friday but sta...|[Stocks, ended, slightly, higher, Friday, staye...|\n","| Assets of the nation's retail money market mut...|[Assets, nations, retail, money, market, mutual...|\n","| Retail sales bounced back a bit in July, and n...|[Retail, sales, bounced, back, bit, July, new, ...|\n","|\" After earning a PH.D. in Sociology, Danny Baz...|[earning, PHD, Sociology, Danny, Bazil, Riley, ...|\n","| Short sellers, Wall Street's dwindling  band o...|[Short, sellers, Wall, Streets, dwindling, band...|\n","| Soaring crude prices plus worries  about the e...|[Soaring, crude, prices, plus, worries, economy...|\n","| OPEC can do nothing to douse scorching  oil pr...|[OPEC, nothing, douse, scorching, oil, prices, ...|\n","| Non OPEC oil exporters should consider  increa...|[Non, OPEC, oil, exporters, consider, increasin...|\n","| WASHINGTON/NEW YORK (Reuters) - The auction fo...|[WASHINGTONNEW, YORK, Reuters, auction, Google,...|\n","| The dollar tumbled broadly on Friday  after da...|[dollar, tumbled, broadly, Friday, data, showin...|\n","|If you think you may need to help your elderly ...|[think, may, need, help, elderly, relatives, fi...|\n","|The purchasing power of kids is a big part of w...|[purchasing, power, kids, big, part, backtoscho...|\n","|There is little cause for celebration in the st...|[little, cause, celebration, stock, market, day...|\n","|The US trade deficit has exploded 19 to a recor...|[US, trade, deficit, exploded, record, bn, oil,...|\n","|Oil giant Shell could be bracing itself for a t...|[Oil, giant, Shell, bracing, takeover, attempt,...|\n","+--------------------------------------------------+--------------------------------------------------+\n","only showing top 20 rows\n","\n"]}],"source":["processed_w2v.select('description','cleanTokens.result').show(truncate=50)"]},{"cell_type":"markdown","metadata":{"id":"-6X5ppwOIPRf"},"source":["## LogReg with Spark NLP Bert Embeddings"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":67199,"status":"ok","timestamp":1711639864590,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"16OpolxP0ClH","outputId":"523a3118-9d5d-4aea-a876-f7a96f4fb4da"},"outputs":[{"output_type":"stream","name":"stdout","text":["bert_base_cased download started this may take some time.\n","Approximate size to download 384.9 MB\n","[OK!]\n"]}],"source":["document_assembler = DocumentAssembler() \\\n","    .setInputCol(\"description\") \\\n","    .setOutputCol(\"document\")\n","\n","tokenizer = Tokenizer() \\\n","    .setInputCols([\"document\"]) \\\n","    .setOutputCol(\"token\")\n","\n","normalizer = Normalizer() \\\n","    .setInputCols([\"token\"]) \\\n","    .setOutputCol(\"normalized\")\n","\n","stopwords_cleaner = StopWordsCleaner()\\\n","    .setInputCols(\"normalized\")\\\n","    .setOutputCol(\"cleanTokens\")\\\n","    .setCaseSensitive(False)\n","\n","bert_embeddings = BertEmbeddings.pretrained('bert_base_cased', 'en') \\\n","    .setInputCols([\"document\",'cleanTokens'])\\\n","    .setOutputCol(\"bert\")\\\n","    .setCaseSensitive(False)\\\n","\n","embeddingsSentence = SentenceEmbeddings() \\\n","    .setInputCols([\"document\", \"bert\"]) \\\n","    .setOutputCol(\"sentence_embeddings\") \\\n","    .setPoolingStrategy(\"AVERAGE\")\n","\n","embeddings_finisher = EmbeddingsFinisher() \\\n","    .setInputCols([\"sentence_embeddings\"]) \\\n","    .setOutputCols([\"finished_sentence_embeddings\"]) \\\n","    .setOutputAsVector(True)\\\n","    .setCleanAnnotations(False)\n","\n","label_stringIdx = StringIndexer(inputCol = \"category\", outputCol = \"label\")\n","\n","\n","nlp_pipeline_bert = Pipeline(\n","    stages=[\n","        document_assembler,\n","        tokenizer,\n","        normalizer,\n","        stopwords_cleaner,\n","        bert_embeddings,\n","        embeddingsSentence,\n","        embeddings_finisher,\n","        label_stringIdx\n","])"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"YScp20Ac8QH4","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711640578045,"user_tz":-60,"elapsed":713468,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"18e04e94-a0f1-4156-def5-9fcf623862a9"},"outputs":[{"output_type":"stream","name":"stdout","text":["CPU times: user 3.75 s, sys: 503 ms, total: 4.25 s\n","Wall time: 11min 53s\n"]},{"output_type":"execute_result","data":{"text/plain":["10000"]},"metadata":{},"execution_count":44}],"source":["%%time\n","limited_df = newsDF.limit(10000)\n","\n","nlp_model_bert = nlp_pipeline_bert.fit(limited_df)\n","\n","processed_bert = nlp_model_bert.transform(limited_df)\n","\n","processed_bert.count()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":11823,"status":"ok","timestamp":1711640589860,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"},"user_tz":-60},"id":"_6YeAaBd0RNp","outputId":"98a86383-1c2b-4a73-f82d-1013936fa79a"},"outputs":[{"output_type":"stream","name":"stdout","text":["+--------------------+--------------------+-----+\n","|         description|            features|label|\n","+--------------------+--------------------+-----+\n","| Short sellers, W...|[-0.0012148499954...|  2.0|\n","| Private investme...|[0.13144001364707...|  2.0|\n","| Soaring crude pr...|[-0.1905521452426...|  2.0|\n","| Authorities have...|[0.06882464885711...|  2.0|\n","| Tearaway world o...|[-0.1174714937806...|  2.0|\n","| Stocks ended sli...|[-0.0321817547082...|  2.0|\n","| Assets of the na...|[-0.2906664609909...|  2.0|\n","| Retail sales bou...|[-0.0385284274816...|  2.0|\n","|\" After earning a...|[-0.0362812764942...|  2.0|\n","| Short sellers, W...|[-0.0012148499954...|  2.0|\n","| Soaring crude pr...|[-0.1905521452426...|  2.0|\n","| OPEC can do noth...|[-0.1431128382682...|  2.0|\n","| Non OPEC oil exp...|[0.01600184850394...|  2.0|\n","| WASHINGTON/NEW Y...|[0.14494352042675...|  2.0|\n","| The dollar tumbl...|[-0.1958881467580...|  2.0|\n","|If you think you ...|[0.27292791008949...|  2.0|\n","|The purchasing po...|[0.00386757543310...|  2.0|\n","|There is little c...|[0.10686706006526...|  2.0|\n","|The US trade defi...|[-0.1425887048244...|  2.0|\n","|Oil giant Shell c...|[0.14418703317642...|  2.0|\n","+--------------------+--------------------+-----+\n","only showing top 20 rows\n","\n"]}],"source":["from pyspark.sql.functions import explode\n","\n","processed_bert = processed_bert.withColumn(\"features\", explode(processed_bert.finished_sentence_embeddings))\n","\n","processed_bert.select('description','features','label').show()\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"EQ1lHAY30RNq","executionInfo":{"status":"ok","timestamp":1711641329182,"user_tz":-60,"elapsed":739331,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"9d3fe800-abdf-4d7f-9b56-a0ff5d7dfdf4"},"outputs":[{"output_type":"stream","name":"stdout","text":["Training Dataset Count: 7033\n","Test Dataset Count: 2967\n"]}],"source":["# set seed for reproducibility\n","(trainingData, testData) = processed_bert.randomSplit([0.7, 0.3], seed = 100)\n","print(\"Training Dataset Count: \" + str(trainingData.count()))\n","print(\"Test Dataset Count: \" + str(testData.count()))"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"7Fhg3kS10RNs"},"outputs":[],"source":["from pyspark.ml.classification import LogisticRegression\n","\n","lr = LogisticRegression(maxIter=20, regParam=0.3, elasticNetParam=0)\n","\n","lrModel = lr.fit(trainingData)\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"rnO-tUfn0RNt"},"outputs":[],"source":["from pyspark.sql.functions import udf\n","\n","@udf(\"long\")\n","def num_nonzeros(v):\n","    return v.numNonzeros()\n","\n","testData = testData.where(num_nonzeros(\"features\") != 0)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"D0BkJHyR0RNt","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711642457327,"user_tz":-60,"elapsed":375030,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"7d1b5dc3-54be-419d-cfc4-b161e6adc975"},"outputs":[{"output_type":"stream","name":"stdout","text":["+------------------------------+--------+------------------------------+-----+----------+\n","|                   description|category|                   probability|label|prediction|\n","+------------------------------+--------+------------------------------+-----+----------+\n","|Wise Solutions has released...|Sci/Tech|[0.9972450751236089,2.61175...|  0.0|       0.0|\n","|Microsoft has a massive pat...|Sci/Tech|[0.9969462971935305,0.00172...|  0.0|       0.0|\n","|Microsoft Corp. has publish...|Sci/Tech|[0.996902163500599,4.952027...|  0.0|       0.0|\n","|A worm that has the capabil...|Sci/Tech|[0.9962258931382776,9.54592...|  0.0|       0.0|\n","|Microsoft has made availabl...|Sci/Tech|[0.9960325952969382,5.09907...|  0.0|       0.0|\n","|Release makes use of techno...|Sci/Tech|[0.9960054388386421,3.32463...|  0.0|       0.0|\n","|Macromedia hopes to boost u...|Sci/Tech|[0.9957959616421305,0.00123...|  0.0|       0.0|\n","|Software giant releases jus...|Sci/Tech|[0.9954924923514694,3.43349...|  0.0|       0.0|\n","| Sleepycat Software this we...|Sci/Tech|[0.9952392415178375,9.38302...|  0.0|       0.0|\n","|On Tuesday, Microsoft unvei...|Sci/Tech|[0.9950684730838191,0.00106...|  0.0|       0.0|\n","+------------------------------+--------+------------------------------+-----+----------+\n","only showing top 10 rows\n","\n"]}],"source":["predictions = lrModel.transform(testData)\n","\n","predictions.select(\"description\",\"category\",\"probability\",\"label\",\"prediction\") \\\n","    .orderBy(\"probability\", ascending=False) \\\n","    .show(n = 10, truncate = 30)\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"6Rn6_IMW0RNt","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711642829945,"user_tz":-60,"elapsed":372620,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"63f46979-b45e-4991-a8df-fe204e44863c"},"outputs":[{"output_type":"stream","name":"stdout","text":["              precision    recall  f1-score   support\n","\n","         0.0       0.84      0.84      0.84       796\n","         1.0       0.87      0.83      0.85       726\n","         2.0       0.81      0.80      0.81       738\n","         3.0       0.89      0.95      0.92       707\n","\n","    accuracy                           0.85      2967\n","   macro avg       0.85      0.85      0.85      2967\n","weighted avg       0.85      0.85      0.85      2967\n","\n","0.8533872598584429\n"]}],"source":["from sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n","import pandas as pd\n","\n","df = predictions.select('description','category','label','prediction').toPandas()\n","\n","print(classification_report(df.label, df.prediction))\n","print(accuracy_score(df.label, df.prediction))"]},{"cell_type":"markdown","metadata":{"id":"tSCxUSkmIPRu"},"source":["## LogReg with ELMO Embeddings"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"L_pCzKheIPRv","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711642884665,"user_tz":-60,"elapsed":54733,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"42f39092-561d-4561-b39e-0f0f3187a01d"},"outputs":[{"output_type":"stream","name":"stdout","text":["elmo download started this may take some time.\n","Approximate size to download 334.1 MB\n","[OK!]\n","CPU times: user 323 ms, sys: 53.5 ms, total: 377 ms\n","Wall time: 54.8 s\n"]},{"output_type":"execute_result","data":{"text/plain":["120000"]},"metadata":{},"execution_count":51}],"source":["%%time\n","\n","document_assembler = DocumentAssembler() \\\n","    .setInputCol(\"description\") \\\n","    .setOutputCol(\"document\")\n","\n","tokenizer = Tokenizer() \\\n","    .setInputCols([\"document\"]) \\\n","    .setOutputCol(\"token\")\n","\n","normalizer = Normalizer() \\\n","    .setInputCols([\"token\"]) \\\n","    .setOutputCol(\"normalized\")\n","\n","stopwords_cleaner = StopWordsCleaner()\\\n","    .setInputCols(\"normalized\")\\\n","    .setOutputCol(\"cleanTokens\")\\\n","    .setCaseSensitive(False)\n","\n","elmo_embeddings = ElmoEmbeddings.pretrained()\\\n","    .setPoolingLayer(\"word_emb\")\\\n","    .setInputCols([\"document\",'cleanTokens'])\\\n","    .setOutputCol(\"elmo\")\n","\n","embeddingsSentence = SentenceEmbeddings() \\\n","    .setInputCols([\"document\", \"elmo\"]) \\\n","    .setOutputCol(\"sentence_embeddings\") \\\n","    .setPoolingStrategy(\"AVERAGE\")\n","\n","embeddings_finisher = EmbeddingsFinisher() \\\n","    .setInputCols([\"sentence_embeddings\"]) \\\n","    .setOutputCols([\"finished_sentence_embeddings\"]) \\\n","    .setOutputAsVector(True)\\\n","    .setCleanAnnotations(False)\n","\n","label_stringIdx = StringIndexer(inputCol = \"category\", outputCol = \"label\")\n","\n","\n","nlp_pipeline_elmo = Pipeline(\n","    stages=[\n","        document_assembler,\n","        tokenizer,\n","        normalizer,\n","        stopwords_cleaner,\n","        elmo_embeddings,\n","        embeddingsSentence,\n","        embeddings_finisher,\n","        label_stringIdx\n","])\n","\n","nlp_model_elmo = nlp_pipeline_elmo.fit(newsDF)\n","\n","processed_elmo = nlp_model_elmo.transform(newsDF)\n","\n","processed_elmo.count()\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"3vk7XGN5IPRx"},"outputs":[],"source":["(trainingData, testData) = newsDF.randomSplit([0.7, 0.3], seed = 100)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"q-VCL5iOIPRz","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711642885587,"user_tz":-60,"elapsed":947,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"b452b3ee-3853-4521-dd1a-5a8778395907"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["83967"]},"metadata":{},"execution_count":53}],"source":["processed_trainingData = nlp_model_elmo.transform(trainingData)\n","\n","processed_trainingData.count()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"QLxq_udOIPR1","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711642885587,"user_tz":-60,"elapsed":7,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"6ad98aee-6a32-4e3e-af03-e5e2c6329d45"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["36033"]},"metadata":{},"execution_count":54}],"source":["processed_testData = nlp_model_elmo.transform(testData)\n","\n","processed_testData.count()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"hWqgBJmKIPR2","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711642885587,"user_tz":-60,"elapsed":6,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"f0d14778-91eb-41d1-b322-f537123362df"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["['category',\n"," 'description',\n"," 'document',\n"," 'token',\n"," 'normalized',\n"," 'cleanTokens',\n"," 'elmo',\n"," 'sentence_embeddings',\n"," 'finished_sentence_embeddings',\n"," 'label']"]},"metadata":{},"execution_count":55}],"source":["processed_trainingData.columns"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"V3W9kceqIPR4"},"outputs":[],"source":["from pyspark.sql.functions import explode\n","\n","processed_testData= processed_testData.withColumn(\"features\", explode(processed_testData.finished_sentence_embeddings))\n","\n","processed_trainingData= processed_trainingData.withColumn(\"features\", explode(processed_trainingData.finished_sentence_embeddings))"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"hY8z7srJIPR6"},"outputs":[],"source":["from pyspark.sql.functions import udf\n","\n","@udf(\"long\")\n","def num_nonzeros(v):\n","    return v.numNonzeros()\n","\n","processed_testData = processed_testData.where(num_nonzeros(\"features\") != 0)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"lyqIOMbsIPR8","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644037959,"user_tz":-60,"elapsed":1152376,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"c855c76b-d260-48dc-a605-c6cec63d1dcf"},"outputs":[{"output_type":"stream","name":"stdout","text":["CPU times: user 7.38 s, sys: 869 ms, total: 8.25 s\n","Wall time: 19min 12s\n"]}],"source":["%%time\n","\n","from pyspark.ml.classification import LogisticRegression\n","\n","lr = LogisticRegression(maxIter=20, regParam=0.3, elasticNetParam=0)\n","\n","lrModel = lr.fit(processed_trainingData)\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"D04iWWPMIPR9","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644037959,"user_tz":-60,"elapsed":7,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"681ad98d-114e-4c4a-acd0-844f79e8fa8b"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["['category',\n"," 'description',\n"," 'document',\n"," 'token',\n"," 'normalized',\n"," 'cleanTokens',\n"," 'elmo',\n"," 'sentence_embeddings',\n"," 'finished_sentence_embeddings',\n"," 'label',\n"," 'features']"]},"metadata":{},"execution_count":59}],"source":["processed_trainingData.columns"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"vS2_xs3ZIPSA","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644288068,"user_tz":-60,"elapsed":250112,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"f6594025-90c0-4ad2-f285-0cf2244de139"},"outputs":[{"output_type":"stream","name":"stdout","text":["+------------------------------+--------+------------------------------+-----+----------+\n","|                   description|category|                   probability|label|prediction|\n","+------------------------------+--------+------------------------------+-----+----------+\n","|\" Costco Wholesale Corp. &l...|Business|[0.995254029013999,0.003926...|  0.0|       0.0|\n","|\" Exxon Mobil Corp. &lt;A H...|Business|[0.9935440409389442,0.00405...|  0.0|       0.0|\n","|\" Exxon Mobil Corp. &lt;A H...|Business|[0.9935440409389442,0.00405...|  0.0|       0.0|\n","| Falling oil prices and str...|   World|[0.9930267682263293,0.00645...|  3.0|       0.0|\n","| ChevronTexaco Corp., the N...|Business|[0.9928294172661469,0.00535...|  0.0|       0.0|\n","| U.S. blue chips fell on We...|Business|[0.9919443128183406,0.00567...|  0.0|       0.0|\n","| Kmart Holding Corporation,...|Business|[0.9914027294702593,0.00687...|  0.0|       0.0|\n","|\" Halliburton Co. &lt;A HRE...|Business|[0.991119081571484,0.004886...|  0.0|       0.0|\n","|\" Halliburton Co. &lt;A HRE...|Business|[0.991119081571484,0.004886...|  0.0|       0.0|\n","|THE stock market is rising,...|Business|[0.990943822857579,0.005011...|  0.0|       0.0|\n","+------------------------------+--------+------------------------------+-----+----------+\n","only showing top 10 rows\n","\n"]}],"source":["predictions = lrModel.transform(processed_testData)\n","\n","predictions.select(\"description\",\"category\",\"probability\",\"label\",\"prediction\") \\\n","    .orderBy(\"probability\", ascending=False) \\\n","    .show(10, truncate = 30)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"P3lz8IFhIPSC"},"outputs":[],"source":["df = predictions.select('description','category','label','prediction').toPandas()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"lrJtqnTcIPSD","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644537705,"user_tz":-60,"elapsed":18,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"46acd73a-233b-4a85-f365-6ba60265fb4f"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["(36033, 4)"]},"metadata":{},"execution_count":62}],"source":["df.shape"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"7eEgmPDWIPSF","colab":{"base_uri":"https://localhost:8080/","height":206},"executionInfo":{"status":"ok","timestamp":1711644537705,"user_tz":-60,"elapsed":15,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"56c5f8fb-9bf6-4573-94ee-0b19b892f1a9"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                         description  category  label  \\\n","0    A drop in oil prices and upbeat outlooks fro...  Business    0.0   \n","1    Air Canada creditors including a General Ele...  Business    0.0   \n","2    AirTran (AAI) said late Thursday it regrets ...  Business    0.0   \n","3    Although published reports yesterday claimed...  Business    0.0   \n","4    Americans paid their credit card bills on ti...  Business    0.0   \n","\n","   prediction  \n","0         0.0  \n","1         0.0  \n","2         3.0  \n","3         0.0  \n","4         0.0  "],"text/html":["\n","  <div id=\"df-0ce8e88e-d8e1-47a7-a08b-1c60e485a9bf\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>description</th>\n","      <th>category</th>\n","      <th>label</th>\n","      <th>prediction</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>A drop in oil prices and upbeat outlooks fro...</td>\n","      <td>Business</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Air Canada creditors including a General Ele...</td>\n","      <td>Business</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>AirTran (AAI) said late Thursday it regrets ...</td>\n","      <td>Business</td>\n","      <td>0.0</td>\n","      <td>3.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Although published reports yesterday claimed...</td>\n","      <td>Business</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Americans paid their credit card bills on ti...</td>\n","      <td>Business</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0ce8e88e-d8e1-47a7-a08b-1c60e485a9bf')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-0ce8e88e-d8e1-47a7-a08b-1c60e485a9bf button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-0ce8e88e-d8e1-47a7-a08b-1c60e485a9bf');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-1e0a3a98-a54e-4497-ada3-1195ea3ca497\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1e0a3a98-a54e-4497-ada3-1195ea3ca497')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-1e0a3a98-a54e-4497-ada3-1195ea3ca497 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df","summary":"{\n  \"name\": \"df\",\n  \"rows\": 36033,\n  \"fields\": [\n    {\n      \"column\": \"description\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 35879,\n        \"samples\": [\n          \"About 29 people, thought to be North Koreans seeking asylum, broke into a Japanese school in Beijing on Wednesday apparently by cutting through the wire perimeter fence.\",\n          \" The dollar finished lower versus major  currencies on Tuesday after weaker than expected U.S. consumer  confidence and regional manufacturing reports opened the door  to some profit taking.\",\n          \"Japan's rate of economic expansion is revised downwards for the three months to June amid falling government investment.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"category\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          \"Sci/Tech\",\n          \"World\",\n          \"Business\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.1191217886461504,\n        \"min\": 0.0,\n        \"max\": 3.0,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          1.0,\n          3.0,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"prediction\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.1162480191097925,\n        \"min\": 0.0,\n        \"max\": 3.0,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          3.0,\n          2.0,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":63}],"source":["df.head()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"6JS3R3knIPSI","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644537705,"user_tz":-60,"elapsed":6,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"34e643d4-ac39-4257-83b4-801347e3d577"},"outputs":[{"output_type":"stream","name":"stdout","text":["              precision    recall  f1-score   support\n","\n","         0.0       0.83      0.83      0.83      9131\n","         1.0       0.84      0.83      0.83      9091\n","         2.0       0.93      0.96      0.94      8880\n","         3.0       0.88      0.87      0.87      8931\n","\n","    accuracy                           0.87     36033\n","   macro avg       0.87      0.87      0.87     36033\n","weighted avg       0.87      0.87      0.87     36033\n","\n","0.870008048178059\n"]}],"source":["from sklearn.metrics import classification_report, accuracy_score\n","\n","print(classification_report(df.label, df.prediction))\n","print(accuracy_score(df.label, df.prediction))"]},{"cell_type":"markdown","metadata":{"id":"8YpFN8zBIPSJ"},"source":["## LogReg with Universal Sentence Encoder"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Tcbxx22GHnel","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644639509,"user_tz":-60,"elapsed":101808,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"b024968a-3334-4765-e543-7c2b579e4019"},"outputs":[{"output_type":"stream","name":"stdout","text":["tfhub_use download started this may take some time.\n","Approximate size to download 923.7 MB\n","[OK!]\n"]}],"source":["useEmbeddings = UniversalSentenceEncoder.pretrained()\\\n","      .setInputCols(\"document\")\\\n","      .setOutputCol(\"use_embeddings\")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"a4YxWiKdIPSK"},"outputs":[],"source":["document_assembler = DocumentAssembler() \\\n","    .setInputCol(\"description\") \\\n","    .setOutputCol(\"document\")\n","\n","loaded_useEmbeddings = UniversalSentenceEncoder.load('/root/cache_pretrained/tfhub_use_en_2.4.0_2.4_1587136330099')\\\n","    .setInputCols(\"document\")\\\n","    .setOutputCol(\"use_embeddings\")\n","\n","embeddings_finisher = EmbeddingsFinisher() \\\n","    .setInputCols([\"use_embeddings\"]) \\\n","    .setOutputCols([\"finished_use_embeddings\"]) \\\n","    .setOutputAsVector(True)\\\n","    .setCleanAnnotations(False)\n","\n","label_stringIdx = StringIndexer(inputCol = \"category\", outputCol = \"label\")\n","\n","use_pipeline = Pipeline(\n","    stages=[\n","        document_assembler,\n","        loaded_useEmbeddings,\n","        embeddings_finisher,\n","        label_stringIdx\n","])\n","\n","use_df = use_pipeline.fit(newsDF).transform(newsDF)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"kbiA3moMIPSR","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644675740,"user_tz":-60,"elapsed":15,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"fcbc8464-23b1-483c-bb1d-d73fbd44b603"},"outputs":[{"output_type":"stream","name":"stdout","text":["+-----------------------+\n","|finished_use_embeddings|\n","+-----------------------+\n","|   [[0.0441501699388...|\n","|   [[0.0844451263546...|\n","|   [[0.0426647029817...|\n","+-----------------------+\n","only showing top 3 rows\n","\n"]}],"source":["use_df.select('finished_use_embeddings').show(3)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"9nI3EXthIPST"},"outputs":[],"source":["from pyspark.sql.functions import explode\n","\n","use_df= use_df.withColumn(\"features\", explode(use_df.finished_use_embeddings))"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"PD9vNF2KIPSV","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644675740,"user_tz":-60,"elapsed":4,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"fcab8dfe-5894-485c-c1bb-fed626cf7a3b"},"outputs":[{"output_type":"stream","name":"stdout","text":["+--------+--------------------+--------------------+--------------------+-----------------------+-----+--------------------+\n","|category|         description|            document|      use_embeddings|finished_use_embeddings|label|            features|\n","+--------+--------------------+--------------------+--------------------+-----------------------+-----+--------------------+\n","|Business| Short sellers, W...|[{document, 0, 84...|[{sentence_embedd...|   [[0.0441501699388...|  0.0|[0.04415016993880...|\n","|Business| Private investme...|[{document, 0, 20...|[{sentence_embedd...|   [[0.0844451263546...|  0.0|[0.08444512635469...|\n","+--------+--------------------+--------------------+--------------------+-----------------------+-----+--------------------+\n","only showing top 2 rows\n","\n"]}],"source":["use_df.show(2)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"2_VHl1FFIPSX","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644716682,"user_tz":-60,"elapsed":40944,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"14aee179-d549-4614-842a-43fdec1e9fc3"},"outputs":[{"output_type":"stream","name":"stdout","text":["Training Dataset Count: 83967\n","Test Dataset Count: 36033\n"]}],"source":["# set seed for reproducibility\n","(trainingData, testData) = use_df.randomSplit([0.7, 0.3], seed = 100)\n","print(\"Training Dataset Count: \" + str(trainingData.count()))\n","print(\"Test Dataset Count: \" + str(testData.count()))"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"vIpDQAScIPSY","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644781907,"user_tz":-60,"elapsed":65254,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"bee59403-a3b7-44fe-9b85-901ed89db173"},"outputs":[{"output_type":"stream","name":"stdout","text":["+------------------------------+--------+------------------------------+-----+----------+\n","|                   description|category|                   probability|label|prediction|\n","+------------------------------+--------+------------------------------+-----+----------+\n","|Amid talk of a possible liq...|Business|[0.9838418958725589,0.00641...|  0.0|       0.0|\n","|Albertsons Inc., one of the...|Business|[0.9828145512629816,0.00860...|  0.0|       0.0|\n","|Prudential Financial Inc., ...|Business|[0.9821409995425612,0.01061...|  0.0|       0.0|\n","|Fannie Mae, the largest US ...|Business|[0.9820438961267436,0.00970...|  0.0|       0.0|\n","|\" Stocks were seen opening ...|Business|[0.9820075348032326,0.00546...|  0.0|       0.0|\n","| U.S. blue chip stocks fell...|Business|[0.9810072366995876,0.01046...|  0.0|       0.0|\n","|Financial services company ...|Business|[0.9806877870270011,0.00947...|  0.0|       0.0|\n","|ExxonMobil Corp. and Royal ...|Business|[0.9806061001659467,0.00650...|  0.0|       0.0|\n","|Finance firm CIT Group Inc....|Business|[0.980403010074237,0.013920...|  0.0|       0.0|\n","|\" U.S. investment bank Morg...|Business|[0.9802807163341225,0.00492...|  0.0|       0.0|\n","+------------------------------+--------+------------------------------+-----+----------+\n","only showing top 10 rows\n","\n"]}],"source":["from sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n","import pandas as pd\n","\n","from pyspark.ml.classification import LogisticRegression\n","\n","lr = LogisticRegression(maxIter=20, regParam=0.3, elasticNetParam=0)\n","\n","lrModel = lr.fit(trainingData)\n","\n","predictions = lrModel.transform(testData)\n","\n","predictions.filter(predictions['prediction'] == 0) \\\n","    .select(\"description\",\"category\",\"probability\",\"label\",\"prediction\") \\\n","    .orderBy(\"probability\", ascending=False) \\\n","    .show(n = 10, truncate = 30)\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"4NiPwWRmIPSa"},"outputs":[],"source":["df = predictions.select('description','category','label','prediction').toPandas()\n","#df['result'] = df['result'].apply(lambda x: x[0])"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"u4Q1XnpZIPSc","colab":{"base_uri":"https://localhost:8080/","height":206},"executionInfo":{"status":"ok","timestamp":1711644801899,"user_tz":-60,"elapsed":21,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"5023f0e9-1207-4b19-8460-f4f279cee9c7"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                         description  category  label  \\\n","0    A drop in oil prices and upbeat outlooks fro...  Business    0.0   \n","1    Air Canada creditors including a General Ele...  Business    0.0   \n","2    AirTran (AAI) said late Thursday it regrets ...  Business    0.0   \n","3    Although published reports yesterday claimed...  Business    0.0   \n","4    Americans paid their credit card bills on ti...  Business    0.0   \n","\n","   prediction  \n","0         0.0  \n","1         0.0  \n","2         0.0  \n","3         0.0  \n","4         0.0  "],"text/html":["\n","  <div id=\"df-e9995a15-1527-4003-a368-3f3d93f5d8b6\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>description</th>\n","      <th>category</th>\n","      <th>label</th>\n","      <th>prediction</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>A drop in oil prices and upbeat outlooks fro...</td>\n","      <td>Business</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Air Canada creditors including a General Ele...</td>\n","      <td>Business</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>AirTran (AAI) said late Thursday it regrets ...</td>\n","      <td>Business</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Although published reports yesterday claimed...</td>\n","      <td>Business</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Americans paid their credit card bills on ti...</td>\n","      <td>Business</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e9995a15-1527-4003-a368-3f3d93f5d8b6')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-e9995a15-1527-4003-a368-3f3d93f5d8b6 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-e9995a15-1527-4003-a368-3f3d93f5d8b6');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-b5a4c9f3-7883-4560-a205-08f2714c3b97\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-b5a4c9f3-7883-4560-a205-08f2714c3b97')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-b5a4c9f3-7883-4560-a205-08f2714c3b97 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df","summary":"{\n  \"name\": \"df\",\n  \"rows\": 36033,\n  \"fields\": [\n    {\n      \"column\": \"description\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 35879,\n        \"samples\": [\n          \"About 29 people, thought to be North Koreans seeking asylum, broke into a Japanese school in Beijing on Wednesday apparently by cutting through the wire perimeter fence.\",\n          \" The dollar finished lower versus major  currencies on Tuesday after weaker than expected U.S. consumer  confidence and regional manufacturing reports opened the door  to some profit taking.\",\n          \"Japan's rate of economic expansion is revised downwards for the three months to June amid falling government investment.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"category\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          \"Sci/Tech\",\n          \"World\",\n          \"Business\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.1191217886461504,\n        \"min\": 0.0,\n        \"max\": 3.0,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          1.0,\n          3.0,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"prediction\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.1169214637802996,\n        \"min\": 0.0,\n        \"max\": 3.0,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          3.0,\n          1.0,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":73}],"source":["df.head()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"aEg_wD3oIPSe","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644801900,"user_tz":-60,"elapsed":8,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"5238a906-ba1f-4096-bcc5-66dc9062edb2"},"outputs":[{"output_type":"stream","name":"stdout","text":["              precision    recall  f1-score   support\n","\n","         0.0       0.83      0.84      0.83      9131\n","         1.0       0.85      0.84      0.85      9091\n","         2.0       0.95      0.97      0.96      8880\n","         3.0       0.89      0.88      0.89      8931\n","\n","    accuracy                           0.88     36033\n","   macro avg       0.88      0.88      0.88     36033\n","weighted avg       0.88      0.88      0.88     36033\n","\n","0.8814142591513335\n"]}],"source":["print(classification_report(df.label, df.prediction))\n","print(accuracy_score(df.label, df.prediction))"]},{"cell_type":"markdown","metadata":{"id":"bdODPjyzIPSf"},"source":["### train on entire dataset"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"LJu8ekkaIPSg"},"outputs":[],"source":["from pyspark.ml.classification import LogisticRegression\n","\n","lr = LogisticRegression(maxIter=20, regParam=0.3, elasticNetParam=0)\n","\n","lrModel = lr.fit(use_df)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"yo8sNiL4IPSh"},"outputs":[],"source":["test_df = spark.read.csv(\"./news_category_test.csv\",header=True)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"22hPNZ9jIPSi"},"outputs":[],"source":["test_df = use_pipeline.fit(test_df).transform(test_df)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"uSJt_s4qIPSj"},"outputs":[],"source":["test_df= test_df.withColumn(\"features\", explode(test_df.finished_use_embeddings))"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"jdwGAMGC2sgP","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644853260,"user_tz":-60,"elapsed":14,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"c722b753-3709-4c59-f6c0-e50754faccee"},"outputs":[{"output_type":"stream","name":"stdout","text":["+--------+--------------------+--------------------+--------------------+-----------------------+-----+--------------------+\n","|category|         description|            document|      use_embeddings|finished_use_embeddings|label|            features|\n","+--------+--------------------+--------------------+--------------------+-----------------------+-----+--------------------+\n","|Business|Unions representi...|[{document, 0, 12...|[{sentence_embedd...|   [[0.0129975201562...|  0.0|[0.01299752015620...|\n","|Sci/Tech| TORONTO, Canada ...|[{document, 0, 22...|[{sentence_embedd...|   [[0.0019998527131...|  1.0|[0.00199985271319...|\n","+--------+--------------------+--------------------+--------------------+-----------------------+-----+--------------------+\n","only showing top 2 rows\n","\n"]}],"source":["test_df.show(2)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"1-1kVlM69X4A","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644858771,"user_tz":-60,"elapsed":5522,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"2f7537ee-6b27-421a-b6a9-a357215c060a"},"outputs":[{"output_type":"stream","name":"stdout","text":["+--------+-----+-----+\n","|category|label|count|\n","+--------+-----+-----+\n","|Sci/Tech|  1.0| 1900|\n","|  Sports|  2.0| 1900|\n","|   World|  3.0| 1900|\n","|Business|  0.0| 1900|\n","+--------+-----+-----+\n","\n"]}],"source":["from pyspark.sql.functions import col\n","\n","test_df.groupBy(\"category\",\"label\") \\\n","    .count() \\\n","    .orderBy(col(\"count\").desc()) \\\n","    .show()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"cG0FjtySIPSm"},"outputs":[],"source":["predictions = lrModel.transform(test_df)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"EnnF27fuIPSn"},"outputs":[],"source":["df = predictions.select('description','category','label','prediction').toPandas()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"daLNQqgAIPSp"},"outputs":[],"source":["df['label'] = df.category.replace({'World':3.0,\n","                    'Sports':2.0,\n","                    'Business':0.0,\n","                    'Sci/Tech':1.0})"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"XxOfQyu322b9","colab":{"base_uri":"https://localhost:8080/","height":206},"executionInfo":{"status":"ok","timestamp":1711644864164,"user_tz":-60,"elapsed":16,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"0fd4b230-d717-4aaf-b5fa-c6652e779fb0"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                         description  category  label  \\\n","0  Unions representing workers at Turner   Newall...  Business    0.0   \n","1   TORONTO, Canada    A second team of rocketeer...  Sci/Tech    1.0   \n","2   A company founded by a chemistry researcher a...  Sci/Tech    1.0   \n","3   It's barely dawn when Mike Fitzpatrick starts...  Sci/Tech    1.0   \n","4   Southern California's smog fighting agency we...  Sci/Tech    1.0   \n","\n","   prediction  \n","0         0.0  \n","1         1.0  \n","2         1.0  \n","3         1.0  \n","4         0.0  "],"text/html":["\n","  <div id=\"df-55f0adff-2e7e-487c-9b62-60dccec0cfb4\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>description</th>\n","      <th>category</th>\n","      <th>label</th>\n","      <th>prediction</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Unions representing workers at Turner   Newall...</td>\n","      <td>Business</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>TORONTO, Canada    A second team of rocketeer...</td>\n","      <td>Sci/Tech</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>A company founded by a chemistry researcher a...</td>\n","      <td>Sci/Tech</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>It's barely dawn when Mike Fitzpatrick starts...</td>\n","      <td>Sci/Tech</td>\n","      <td>1.0</td>\n","      <td>1.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Southern California's smog fighting agency we...</td>\n","      <td>Sci/Tech</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-55f0adff-2e7e-487c-9b62-60dccec0cfb4')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-55f0adff-2e7e-487c-9b62-60dccec0cfb4 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-55f0adff-2e7e-487c-9b62-60dccec0cfb4');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-c017c919-30a6-414a-8f73-775e5c5a1157\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c017c919-30a6-414a-8f73-775e5c5a1157')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-c017c919-30a6-414a-8f73-775e5c5a1157 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df","summary":"{\n  \"name\": \"df\",\n  \"rows\": 7600,\n  \"fields\": [\n    {\n      \"column\": \"description\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 7592,\n        \"samples\": [\n          \" Second seeded Lleyton Hewitt reached Sunday #39;s final at the  $380,000 TD Waterhouse Cup    a final US Open tune up.\",\n          \"Crude futures eased slightly Thursday after a US government report showed another boost in supplies ahead of the Northern Hemisphere winter.\",\n          \"Three men were arrested Tuesday night for trying to extort \\\\$3 million from Denver Nuggets star Carmelo Anthony. Joubert Santos and Jason Pabon of the Bronx \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"category\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          \"Sci/Tech\",\n          \"World\",\n          \"Business\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.1181075508770757,\n        \"min\": 0.0,\n        \"max\": 3.0,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          1.0,\n          3.0,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"prediction\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.1119434380667452,\n        \"min\": 0.0,\n        \"max\": 3.0,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          1.0,\n          3.0,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":84}],"source":["df.head()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"QqVmtiew81e6","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644864164,"user_tz":-60,"elapsed":14,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"3e99b7a0-fc59-4180-8eb4-f2bf74daf269"},"outputs":[{"output_type":"stream","name":"stdout","text":["              precision    recall  f1-score   support\n","\n","         0.0       0.83      0.83      0.83      1900\n","         1.0       0.84      0.85      0.85      1900\n","         2.0       0.95      0.97      0.96      1900\n","         3.0       0.90      0.87      0.89      1900\n","\n","    accuracy                           0.88      7600\n","   macro avg       0.88      0.88      0.88      7600\n","weighted avg       0.88      0.88      0.88      7600\n","\n","0.8802631578947369\n"]}],"source":["from sklearn.metrics import classification_report, accuracy_score\n","\n","print(classification_report(df.label, df.prediction))\n","print(accuracy_score(df.label, df.prediction))"]},{"cell_type":"markdown","metadata":{"id":"regAngWEIPTB"},"source":["# ClassifierDL"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"8RfV7lB_IPTB"},"outputs":[],"source":["# actual content is inside description column\n","document = DocumentAssembler()\\\n","    .setInputCol(\"description\")\\\n","    .setOutputCol(\"document\")\n","\n","use = UniversalSentenceEncoder.load('/root/cache_pretrained/tfhub_use_en_2.4.0_2.4_1587136330099')\\\n","    .setInputCols(\"document\")\\\n","    .setOutputCol(\"sentence_embeddings\")\n","\n","# the classes/labels/categories are in category column\n","classsifierdl = ClassifierDLApproach()\\\n","    .setInputCols([\"sentence_embeddings\"])\\\n","    .setOutputCol(\"class\")\\\n","    .setLabelColumn(\"category\")\\\n","    .setMaxEpochs(5)\\\n","    .setEnableOutputLogs(True)\n","\n","pipeline = Pipeline(\n","    stages = [\n","        document,\n","        use,\n","        classsifierdl\n","])"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"hfsmr81a43Pz","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644884448,"user_tz":-60,"elapsed":575,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"b3dec277-d725-43cc-e4f3-9f33709a5ee1"},"outputs":[{"output_type":"stream","name":"stdout","text":["Training Dataset Count: 83967\n","Test Dataset Count: 36033\n"]}],"source":["# set seed for reproducibility\n","(trainingData, testData) = newsDF.randomSplit([0.7, 0.3], seed = 100)\n","print(\"Training Dataset Count: \" + str(trainingData.count()))\n","print(\"Test Dataset Count: \" + str(testData.count()))"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"H47jOwwCIPTG"},"outputs":[],"source":["pipelineModel = pipeline.fit(trainingData)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"ZHQj7chI5JM5"},"outputs":[],"source":["pipelineModel.stages[2].write().overwrite().save('classifierDL_model_5e')"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"_95KHyT349ZO","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644968150,"user_tz":-60,"elapsed":14237,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"ea4fcddf-045d-4290-fb5f-199ea1a7a44b"},"outputs":[{"output_type":"stream","name":"stdout","text":["              precision    recall  f1-score   support\n","\n","    Business       0.84      0.87      0.85      9131\n","    Sci/Tech       0.86      0.87      0.86      9091\n","      Sports       0.95      0.98      0.96      8880\n","       World       0.93      0.86      0.89      8931\n","\n","    accuracy                           0.89     36033\n","   macro avg       0.89      0.89      0.89     36033\n","weighted avg       0.89      0.89      0.89     36033\n","\n","0.8929037271390114\n"]}],"source":["from sklearn.metrics import classification_report, accuracy_score\n","\n","df = pipelineModel.transform(testData).select('category','description',\"class.result\").toPandas()\n","\n","df['result'] = df['result'].apply(lambda x: x[0])\n","\n","print(classification_report(df.category, df.result))\n","print(accuracy_score(df.category, df.result))"]},{"cell_type":"markdown","metadata":{"id":"GkeFxX8tIPTJ"},"source":["## Loading the trained classifier from disk"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"ACf2NVAkIPTK","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644968151,"user_tz":-60,"elapsed":18,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"57766d08-8bff-4d00-e5f3-cfb3426848f2"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["['/usr/local/lib/python3.10/dist-packages/sparknlp']"]},"metadata":{},"execution_count":91}],"source":["import sparknlp\n","sparknlp.__path__"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"vS0TSHm9IPTM"},"outputs":[],"source":["trainDataset = spark.read \\\n","      .option(\"header\", True) \\\n","      .csv(\"news_category_train.csv\")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"ykBB6BW-59so","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644968151,"user_tz":-60,"elapsed":12,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"3b43af5d-b0c8-4c85-a235-1bc82d4c253c"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["120000"]},"metadata":{},"execution_count":93}],"source":["trainDataset.count()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"OdZ2YoLE6Bro","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644968805,"user_tz":-60,"elapsed":665,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"ee0b0bdc-4cbe-43f6-d0a5-5ffcc542e945"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["83967"]},"metadata":{},"execution_count":94}],"source":["trainingData.count()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"ZkStUCiJIPTR"},"outputs":[],"source":["document = DocumentAssembler()\\\n","    .setInputCol(\"description\")\\\n","    .setOutputCol(\"document\")\n","\n","use = UniversalSentenceEncoder.load('/root/cache_pretrained/tfhub_use_en_2.4.0_2.4_1587136330099')\\\n","    .setInputCols(\"document\")\\\n","    .setOutputCol(\"sentence_embeddings\")\n","\n","classsifierdlmodel = ClassifierDLModel.load('classifierDL_model_5e')\n","\n","pipeline = Pipeline(\n","    stages = [\n","        document,\n","        use,\n","        classsifierdlmodel\n","])"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"-jlVnXBC6sDZ","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644990897,"user_tz":-60,"elapsed":2222,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"ffd30ade-0e8a-49b1-a79b-83e647ef7f65"},"outputs":[{"output_type":"stream","name":"stdout","text":["+--------+--------------------------------------------------+----------+\n","|category|                                       description|    result|\n","+--------+--------------------------------------------------+----------+\n","|Business|  A drop in oil prices and upbeat outlooks from...|[Business]|\n","|Business|  Air Canada creditors including a General Elec...|[Business]|\n","|Business|  AirTran (AAI) said late Thursday it regrets t...|[Business]|\n","|Business|  Although published reports yesterday claimed ...|[Business]|\n","|Business|  Americans paid their credit card bills on tim...|[Business]|\n","|Business|  As Apple Computer shares trade near their hig...|[Sci/Tech]|\n","|Business|  Asian stocks closed mainly higher Tuesday, le...|[Business]|\n","|Business|  Bank of America chief executive Ken Lewis wil...|[Business]|\n","|Business|  Big gains in technology stocks helped the Nas...|[Business]|\n","|Business|  Cargo operations at the nation's largest port...|[Business]|\n","+--------+--------------------------------------------------+----------+\n","\n"]}],"source":["pipeline.fit(testData.limit(10)).transform(testData.limit(10)).select('category','description',\"class.result\").show(10, truncate=50)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"sByX3oVm6vKn","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644990897,"user_tz":-60,"elapsed":7,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"9b914e28-e4eb-4250-ac91-403222ed9c0f"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'document': ['In its first two years, the UK dedicated card companies have surge'],\n"," 'sentence_embeddings': ['In its first two years, the UK dedicated card companies have surge'],\n"," 'class': ['Sci/Tech']}"]},"metadata":{},"execution_count":97}],"source":["lm = LightPipeline(pipeline.fit(spark.createDataFrame([[\"\"]]).toDF(\"text\")))\n","lm.annotate('In its first two years, the UK dedicated card companies have surge')"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"pzdks_VjIPTV"},"outputs":[],"source":["text='''\n","Fearing the fate of Italy, the centre-right government has threatened to be merciless with those who flout tough restrictions. As of Wednesday it will also include all shops being closed across Greece, with the exception of supermarkets. Banks, pharmacies, pet-stores, mobile phone stores, opticians, bakers, mini-markets, couriers and food delivery outlets are among the few that will also be allowed to remain open.\n","'''"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"mUEhBk1h6yMR","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711644990897,"user_tz":-60,"elapsed":4,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"58e89efe-4899-4ffa-db62-a7eb57d8c3dc"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'document': ['\\nFearing the fate of Italy, the centre-right government has threatened to be merciless with those who flout tough restrictions. As of Wednesday it will also include all shops being closed across Greece, with the exception of supermarkets. Banks, pharmacies, pet-stores, mobile phone stores, opticians, bakers, mini-markets, couriers and food delivery outlets are among the few that will also be allowed to remain open.\\n'],\n"," 'sentence_embeddings': ['\\nFearing the fate of Italy, the centre-right government has threatened to be merciless with those who flout tough restrictions. As of Wednesday it will also include all shops being closed across Greece, with the exception of supermarkets. Banks, pharmacies, pet-stores, mobile phone stores, opticians, bakers, mini-markets, couriers and food delivery outlets are among the few that will also be allowed to remain open.\\n'],\n"," 'class': ['Business']}"]},"metadata":{},"execution_count":99}],"source":["lm = LightPipeline(pipeline.fit(spark.createDataFrame([[\"\"]]).toDF(\"text\")))\n","\n","lm.annotate(text)"]},{"cell_type":"markdown","metadata":{"id":"_vn6mkWb-QbH"},"source":["# Classifier DL + Glove + Basic text processing"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"0KV-gdcL-QbI","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711645006671,"user_tz":-60,"elapsed":15777,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"f5a84476-73cf-4da3-f2c3-5642d0e778f8"},"outputs":[{"output_type":"stream","name":"stdout","text":["lemma_antbnc download started this may take some time.\n","Approximate size to download 907.6 KB\n","[OK!]\n","glove_100d download started this may take some time.\n","Approximate size to download 145.3 MB\n","[OK!]\n"]}],"source":["document = DocumentAssembler()\\\n","    .setInputCol(\"description\")\\\n","    .setOutputCol(\"document\")\n","\n","tokenizer = Tokenizer() \\\n","    .setInputCols([\"document\"]) \\\n","    .setOutputCol(\"token\")\n","\n","lemma = LemmatizerModel.pretrained('lemma_antbnc') \\\n","    .setInputCols([\"token\"]) \\\n","    .setOutputCol(\"lemma\")\n","\n","glove_embeddings = WordEmbeddingsModel().pretrained() \\\n","    .setInputCols([\"document\",'lemma'])\\\n","    .setOutputCol(\"embeddings\")\\\n","    .setCaseSensitive(False)\n","\n","lemma_pipeline = Pipeline(\n","    stages=[\n","        document,\n","        tokenizer,\n","        lemma,\n","        glove_embeddings\n","])"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"m1Up3FlT-QbK","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711645007867,"user_tz":-60,"elapsed":1207,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"dc137251-a6d0-4030-ada4-91f2f75ce8fd"},"outputs":[{"output_type":"stream","name":"stdout","text":["+--------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+\n","|category|                   description|                      document|                         token|                         lemma|                    embeddings|\n","+--------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+\n","|Business|  A  $120 million fine levi...|[{document, 0, 278,   A  $1...|[{token, 2, 2, A, {sentence...|[{token, 2, 2, A, {sentence...|[{word_embeddings, 2, 2, A,...|\n","|Business|  A Colorado assistant stor...|[{document, 0, 144,   A Col...|[{token, 2, 2, A, {sentence...|[{token, 2, 2, A, {sentence...|[{word_embeddings, 2, 2, A,...|\n","|Business|  A Pennsylvania brewery is...|[{document, 0, 132,   A Pen...|[{token, 2, 2, A, {sentence...|[{token, 2, 2, A, {sentence...|[{word_embeddings, 2, 2, A,...|\n","|Business|  A federal judge has sched...|[{document, 0, 140,   A fed...|[{token, 2, 2, A, {sentence...|[{token, 2, 2, A, {sentence...|[{word_embeddings, 2, 2, A,...|\n","|Business|  A group led by privately ...|[{document, 0, 186,   A gro...|[{token, 2, 2, A, {sentence...|[{token, 2, 2, A, {sentence...|[{word_embeddings, 2, 2, A,...|\n","|Business|  A special committee of Ho...|[{document, 0, 198,   A spe...|[{token, 2, 2, A, {sentence...|[{token, 2, 2, A, {sentence...|[{word_embeddings, 2, 2, A,...|\n","|Business|  A string of government re...|[{document, 0, 84,   A stri...|[{token, 2, 2, A, {sentence...|[{token, 2, 2, A, {sentence...|[{word_embeddings, 2, 2, A,...|\n","|Business|  AMP Ltd., Australia #39;s...|[{document, 0, 228,   AMP L...|[{token, 2, 4, AMP, {senten...|[{token, 2, 4, AMP, {senten...|[{word_embeddings, 2, 4, AM...|\n","|Business|  After months of unprecede...|[{document, 0, 193,   After...|[{token, 2, 6, After, {sent...|[{token, 2, 6, After, {sent...|[{word_embeddings, 2, 6, Af...|\n","|Business|  Airbus on Friday said it ...|[{document, 0, 113,   Airbu...|[{token, 2, 7, Airbus, {sen...|[{token, 2, 7, Airbus, {sen...|[{word_embeddings, 2, 7, Ai...|\n","|Business|  Alcoa, Diversified Utilit...|[{document, 0, 165,   Alcoa...|[{token, 2, 6, Alcoa, {sent...|[{token, 2, 6, Alcoa, {sent...|[{word_embeddings, 2, 6, Al...|\n","|Business|  An Ontario court approved...|[{document, 0, 145,   An On...|[{token, 2, 3, An, {sentenc...|[{token, 2, 3, An, {sentenc...|[{word_embeddings, 2, 3, An...|\n","|Business|  Andrew Mohl (left), chief...|[{document, 0, 240,   Andre...|[{token, 2, 7, Andrew, {sen...|[{token, 2, 7, Andrew, {sen...|[{word_embeddings, 2, 7, An...|\n","|Business|  Applied Materials beat Wa...|[{document, 0, 232,   Appli...|[{token, 2, 8, Applied, {se...|[{token, 2, 8, Applied, {se...|[{word_embeddings, 2, 8, Ap...|\n","|Business|  At least for now, the slo...|[{document, 0, 140,   At le...|[{token, 2, 3, At, {sentenc...|[{token, 2, 3, At, {sentenc...|[{word_embeddings, 2, 3, At...|\n","|Business|  BHP Billiton, the world #...|[{document, 0, 227,   BHP B...|[{token, 2, 4, BHP, {senten...|[{token, 2, 4, BHP, {senten...|[{word_embeddings, 2, 4, BH...|\n","|Business|  BHP Billiton, the world #...|[{document, 0, 229,   BHP B...|[{token, 2, 4, BHP, {senten...|[{token, 2, 4, BHP, {senten...|[{word_embeddings, 2, 4, BH...|\n","|Business|  Banco Santander Central H...|[{document, 0, 237,   Banco...|[{token, 2, 6, Banco, {sent...|[{token, 2, 6, Banco, {sent...|[{word_embeddings, 2, 6, Ba...|\n","|Business|  Barclays Plc, the UK #39;...|[{document, 0, 211,   Barcl...|[{token, 2, 9, Barclays, {s...|[{token, 2, 9, Barclays, {s...|[{word_embeddings, 2, 9, Ba...|\n","|Business|  Barclays Plc, the UK #39;...|[{document, 0, 217,   Barcl...|[{token, 2, 9, Barclays, {s...|[{token, 2, 9, Barclays, {s...|[{word_embeddings, 2, 9, Ba...|\n","+--------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+\n","only showing top 20 rows\n","\n"]}],"source":["lemma_pipeline.fit(trainingData.limit(1000)).transform(trainingData.limit(1000)).show(truncate=30)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"kXpQOv1J-QbL","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711645016553,"user_tz":-60,"elapsed":8691,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"c9938710-2843-4cbf-f36c-1f7e28f62511"},"outputs":[{"output_type":"stream","name":"stdout","text":["lemma_antbnc download started this may take some time.\n","Approximate size to download 907.6 KB\n","[OK!]\n","glove_100d download started this may take some time.\n","Approximate size to download 145.3 MB\n","[OK!]\n"]}],"source":["document_assembler = DocumentAssembler() \\\n","    .setInputCol(\"description\") \\\n","    .setOutputCol(\"document\")\n","\n","tokenizer = Tokenizer() \\\n","    .setInputCols([\"document\"]) \\\n","    .setOutputCol(\"token\")\n","\n","normalizer = Normalizer() \\\n","    .setInputCols([\"token\"]) \\\n","    .setOutputCol(\"normalized\")\n","\n","stopwords_cleaner = StopWordsCleaner()\\\n","    .setInputCols(\"normalized\")\\\n","    .setOutputCol(\"cleanTokens\")\\\n","    .setCaseSensitive(False)\n","\n","lemma = LemmatizerModel.pretrained('lemma_antbnc') \\\n","    .setInputCols([\"cleanTokens\"]) \\\n","    .setOutputCol(\"lemma\")\n","\n","glove_embeddings = WordEmbeddingsModel().pretrained() \\\n","    .setInputCols([\"document\",'lemma'])\\\n","    .setOutputCol(\"embeddings\")\\\n","    .setCaseSensitive(False)\n","\n","embeddingsSentence = SentenceEmbeddings() \\\n","    .setInputCols([\"document\", \"embeddings\"]) \\\n","    .setOutputCol(\"sentence_embeddings\") \\\n","    .setPoolingStrategy(\"AVERAGE\")\n","\n","classsifierdl = ClassifierDLApproach()\\\n","    .setInputCols([\"sentence_embeddings\"])\\\n","    .setOutputCol(\"class\")\\\n","    .setLabelColumn(\"category\")\\\n","    .setMaxEpochs(5)\\\n","    .setEnableOutputLogs(True)\n","\n","clf_pipeline = Pipeline(\n","    stages=[\n","        document_assembler,\n","        tokenizer,\n","        normalizer,\n","        stopwords_cleaner,\n","        lemma,\n","        glove_embeddings,\n","        embeddingsSentence,\n","        classsifierdl\n","])"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"_CEiY2qW-QbM"},"outputs":[],"source":["!rm -rf classifier_dl_pipeline_glove"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"4Ru_yFKQ-QbN"},"outputs":[],"source":["clf_pipeline.save('classifier_dl_pipeline_glove')"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"R8tDkEtn-QbN"},"outputs":[],"source":["clf_pipelineModel = clf_pipeline.fit(trainingData)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"4tt10fAn-QbO","colab":{"base_uri":"https://localhost:8080/","height":206},"executionInfo":{"status":"ok","timestamp":1711645120295,"user_tz":-60,"elapsed":26625,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"9813ca30-d131-4dfd-b365-1a5f6681f95e"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["   category                                        description      result\n","0  Business    A drop in oil prices and upbeat outlooks fro...  [Business]\n","1  Business    Air Canada creditors including a General Ele...  [Business]\n","2  Business    AirTran (AAI) said late Thursday it regrets ...  [Business]\n","3  Business    Although published reports yesterday claimed...  [Business]\n","4  Business    Americans paid their credit card bills on ti...  [Business]"],"text/html":["\n","  <div id=\"df-c9652c87-118f-4c78-89be-6d22928cf0a9\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>description</th>\n","      <th>result</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Business</td>\n","      <td>A drop in oil prices and upbeat outlooks fro...</td>\n","      <td>[Business]</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Business</td>\n","      <td>Air Canada creditors including a General Ele...</td>\n","      <td>[Business]</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Business</td>\n","      <td>AirTran (AAI) said late Thursday it regrets ...</td>\n","      <td>[Business]</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Business</td>\n","      <td>Although published reports yesterday claimed...</td>\n","      <td>[Business]</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Business</td>\n","      <td>Americans paid their credit card bills on ti...</td>\n","      <td>[Business]</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c9652c87-118f-4c78-89be-6d22928cf0a9')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-c9652c87-118f-4c78-89be-6d22928cf0a9 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-c9652c87-118f-4c78-89be-6d22928cf0a9');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-22ba2af2-edfc-40d0-be05-25e98b1558e9\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-22ba2af2-edfc-40d0-be05-25e98b1558e9')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-22ba2af2-edfc-40d0-be05-25e98b1558e9 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df","summary":"{\n  \"name\": \"df\",\n  \"rows\": 36033,\n  \"fields\": [\n    {\n      \"column\": \"category\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          \"Sci/Tech\",\n          \"World\",\n          \"Business\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"description\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 35879,\n        \"samples\": [\n          \"About 29 people, thought to be North Koreans seeking asylum, broke into a Japanese school in Beijing on Wednesday apparently by cutting through the wire perimeter fence.\",\n          \" The dollar finished lower versus major  currencies on Tuesday after weaker than expected U.S. consumer  confidence and regional manufacturing reports opened the door  to some profit taking.\",\n          \"Japan's rate of economic expansion is revised downwards for the three months to June amid falling government investment.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"result\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":106}],"source":["df = clf_pipelineModel.transform(testData).select('category','description',\"class.result\").toPandas()\n","\n","df.head()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"MfqT1kOExFyD","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711645121578,"user_tz":-60,"elapsed":1286,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"301d46e8-1df3-4bd7-be30-9ed0c16faf98"},"outputs":[{"output_type":"stream","name":"stdout","text":["              precision    recall  f1-score   support\n","\n","    Business       0.84      0.85      0.84      9131\n","    Sci/Tech       0.84      0.86      0.85      9091\n","      Sports       0.94      0.96      0.95      8880\n","       World       0.91      0.86      0.89      8931\n","\n","    accuracy                           0.88     36033\n","   macro avg       0.88      0.88      0.88     36033\n","weighted avg       0.88      0.88      0.88     36033\n","\n","0.8832736658063441\n"]}],"source":["from sklearn.metrics import classification_report, accuracy_score\n","\n","df['result'] = df['result'].apply(lambda x: x[0])\n","\n","print(classification_report(df.category, df.result))\n","\n","print(accuracy_score(df.category, df.result))"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"jych1Rk8-QbQ"},"outputs":[],"source":["import pandas as pd"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"pie9T7en-QbQ"},"outputs":[],"source":["news_df = newsDF.toPandas()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"AGJEumf--QbR","colab":{"base_uri":"https://localhost:8080/","height":206},"executionInfo":{"status":"ok","timestamp":1711645122838,"user_tz":-60,"elapsed":9,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"cd4739af-0f00-4b9b-8928-8c5dd984b82c"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["   category                                        description\n","0  Business   Short sellers, Wall Street's dwindling band o...\n","1  Business   Private investment firm Carlyle Group, which ...\n","2  Business   Soaring crude prices plus worries about the e...\n","3  Business   Authorities have halted oil export flows from...\n","4  Business   Tearaway world oil prices, toppling records a..."],"text/html":["\n","  <div id=\"df-da550f2f-9eb1-4e66-b7a9-b601c9b76af8\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>description</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Business</td>\n","      <td>Short sellers, Wall Street's dwindling band o...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Business</td>\n","      <td>Private investment firm Carlyle Group, which ...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Business</td>\n","      <td>Soaring crude prices plus worries about the e...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Business</td>\n","      <td>Authorities have halted oil export flows from...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Business</td>\n","      <td>Tearaway world oil prices, toppling records a...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-da550f2f-9eb1-4e66-b7a9-b601c9b76af8')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-da550f2f-9eb1-4e66-b7a9-b601c9b76af8 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-da550f2f-9eb1-4e66-b7a9-b601c9b76af8');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-d191366d-6157-4110-b8da-c4caa9c80abe\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-d191366d-6157-4110-b8da-c4caa9c80abe')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-d191366d-6157-4110-b8da-c4caa9c80abe button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"news_df"}},"metadata":{},"execution_count":110}],"source":["news_df.head()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"7gFIeTJw-QbS"},"outputs":[],"source":["news_df.to_csv('news_dataset.csv', index=False)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"HT0IKCms-QbT","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1711645134086,"user_tz":-60,"elapsed":10751,"user":{"displayName":"Akar Öztürk","userId":"07602883577262601067"}},"outputId":"95066d7b-4f4c-4d8c-87f3-a3b1d439c9c8"},"outputs":[{"output_type":"stream","name":"stdout","text":["lemma_antbnc download started this may take some time.\n","Approximate size to download 907.6 KB\n","[OK!]\n","glove_100d download started this may take some time.\n","Approximate size to download 145.3 MB\n","[OK!]\n"]}],"source":["document_assembler = DocumentAssembler() \\\n","    .setInputCol(\"description\") \\\n","    .setOutputCol(\"document\")\n","\n","tokenizer = Tokenizer() \\\n","    .setInputCols([\"document\"]) \\\n","    .setOutputCol(\"token\")\n","\n","normalizer = Normalizer() \\\n","    .setInputCols([\"token\"]) \\\n","    .setOutputCol(\"normalized\")\n","\n","stopwords_cleaner = StopWordsCleaner()\\\n","    .setInputCols(\"normalized\")\\\n","    .setOutputCol(\"cleanTokens\")\\\n","    .setCaseSensitive(False)\n","\n","lemma = LemmatizerModel.pretrained('lemma_antbnc') \\\n","    .setInputCols([\"cleanTokens\"]) \\\n","    .setOutputCol(\"lemma\")\n","\n","glove_embeddings = WordEmbeddingsModel().pretrained() \\\n","    .setInputCols([\"document\",'lemma'])\\\n","    .setOutputCol(\"embeddings\")\\\n","    .setCaseSensitive(False)\n","\n","txt_pipeline = Pipeline(\n","    stages=[\n","        document_assembler,\n","        tokenizer,\n","        normalizer,\n","        stopwords_cleaner,\n","        lemma,\n","        glove_embeddings,\n","        embeddingsSentence\n","])"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"ZxUrgGoU-QbU"},"outputs":[],"source":["txt_pipelineModel = txt_pipeline.fit(testData.limit(1))"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"BJqR-_Gu-QbU"},"outputs":[],"source":["txt_pipelineModel.save('text_prep_pipeline_glove')"]}],"metadata":{"accelerator":"GPU","colab":{"machine_shape":"hm","provenance":[],"gpuType":"T4"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.4"}},"nbformat":4,"nbformat_minor":0}