From ce09a72cbd19a3d2af3bc1586ac21f1836d361f4 Mon Sep 17 00:00:00 2001 From: luxu1 Date: Mon, 13 Dec 2021 14:04:15 -0800 Subject: [PATCH] modify for new pr --- .../computed-col-test.ipynb | 154 +++++++++--------- 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/samples/computed-col-test-cases/computed-col-test.ipynb b/samples/computed-col-test-cases/computed-col-test.ipynb index 30a496e..2d1ae45 100644 --- a/samples/computed-col-test-cases/computed-col-test.ipynb +++ b/samples/computed-col-test-cases/computed-col-test.ipynb @@ -50,19 +50,20 @@ }, "outputs": [ { + "output_type": "stream", "name": "stdout", - "text": "Starting Spark application\n", - "output_type": "stream" + "text": "Starting Spark application\n" }, { + "output_type": "display_data", "data": { "text/plain": "", "text/html": "\n
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
32application_1628887230618_0066pysparkidleLinkLink✔
" }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -71,15 +72,15 @@ "model_id": "57594e6996e94f2bb9181dee19ddaafa" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "SparkSession available as 'spark'.\n", - "output_type": "stream" + "text": "SparkSession available as 'spark'.\n" }, { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -88,8 +89,7 @@ "model_id": "49029d8d1d164fd0966752ecdf3aa4d5" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} } ], "execution_count": 2 @@ -124,6 +124,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -132,13 +133,12 @@ "model_id": "b0b0e4f5ed664b71aae1a27adc41ddac" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+---+----+-----+\n| Id|Year|Month|\n+---+----+-----+\n| 1|2020| 01|\n| 2|2020| 02|\n| 3|2020| 03|\n+---+----+-----+\n\n+------+------------+--------------------+\n|PostID| PostTitle| PostBody|\n+------+------------+--------------------+\n| 1| Intro|Hi There This is ABC|\n| 2| Intro| Hello Im PQR|\n| 3| Re: Intro| Hey PQR This is XYZ|\n| 4| Geography| Im George from USA|\n| 5|Re:Geography| Im Mary from OZ|\n| 6|Re:Geography| Im Peter from UK|\n+------+------------+--------------------+", - "output_type": "stream" + "text": "+---+----+-----+\n| Id|Year|Month|\n+---+----+-----+\n| 1|2020| 01|\n| 2|2020| 02|\n| 3|2020| 03|\n+---+----+-----+\n\n+------+------------+--------------------+\n|PostID| PostTitle| PostBody|\n+------+------------+--------------------+\n| 1| Intro|Hi There This is ABC|\n| 2| Intro| Hello Im PQR|\n| 3| Re: Intro| Hey PQR This is XYZ|\n| 4| Geography| Im George from USA|\n| 5|Re:Geography| Im Mary from OZ|\n| 6|Re:Geography| Im Peter from UK|\n+------+------------+--------------------+" } ], "execution_count": 5 @@ -168,7 +168,7 @@ " .option(\"dbtable\", dbtable) \\\r\n", " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", - " .option(\"schemaCheckEnabled\", False) \\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -185,6 +185,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -193,13 +194,12 @@ "model_id": "ef11011fce0a4d86b2ab67d3506ca6d0" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 6 @@ -229,6 +229,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -237,13 +238,12 @@ "model_id": "3d68cffa23f74740b5c234aec7036761" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+-------+---+----+-----+--------+\n| Date| Id|Year|Month| Years|\n+-------+---+----+-----+--------+\n|2020-02| 2|2020| 02|20202020|\n|2020-03| 3|2020| 03|20202020|\n|2020-01| 1|2020| 01|20202020|\n+-------+---+----+-----+--------+", - "output_type": "stream" + "text": "+-------+---+----+-----+--------+\n| Date| Id|Year|Month| Years|\n+-------+---+----+-----+--------+\n|2020-02| 2|2020| 02|20202020|\n|2020-03| 3|2020| 03|20202020|\n|2020-01| 1|2020| 01|20202020|\n+-------+---+----+-----+--------+" } ], "execution_count": 7 @@ -263,7 +263,7 @@ " .option(\"dbtable\", dbtable) \\\r\n", " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", - " .option(\"schemaCheckEnabled\", False) \\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -281,6 +281,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -289,13 +290,12 @@ "model_id": "d0f1fb3556824b68acf73d1ef0cc4d6b" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 8 @@ -325,6 +325,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -333,13 +334,12 @@ "model_id": "83f7f1511dd14c26a6b63053fac7a2d3" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+---+-------+----+-----+\n| Id| Date|Year|Month|\n+---+-------+----+-----+\n| 1|2020-01|2020| 01|\n| 2|2020-02|2020| 02|\n| 3|2020-03|2020| 03|\n+---+-------+----+-----+", - "output_type": "stream" + "text": "+---+-------+----+-----+\n| Id| Date|Year|Month|\n+---+-------+----+-----+\n| 1|2020-01|2020| 01|\n| 2|2020-02|2020| 02|\n| 3|2020-03|2020| 03|\n+---+-------+----+-----+" } ], "execution_count": 9 @@ -359,7 +359,7 @@ " .option(\"dbtable\", dbtable) \\\r\n", " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", - " .option(\"schemaCheckEnabled\", False) \\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -377,6 +377,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -385,13 +386,12 @@ "model_id": "880a7b25e6fc472dac6ca47f737ffdda" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) done", - "output_type": "stream" + "text": "MSSQL Connector write(append) done" } ], "execution_count": 10 @@ -422,6 +422,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -430,13 +431,12 @@ "model_id": "94f5c08789b849ffa6e7469d0ccb9e34" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+---+-------+----+--------+-----+\n| Id| Date|Year| Years|Month|\n+---+-------+----+--------+-----+\n| 2|2020-02|2020|20202020| 02|\n| 3|2020-03|2020|20202020| 03|\n| 1|2020-01|2020|20202020| 01|\n+---+-------+----+--------+-----+", - "output_type": "stream" + "text": "+---+-------+----+--------+-----+\n| Id| Date|Year| Years|Month|\n+---+-------+----+--------+-----+\n| 2|2020-02|2020|20202020| 02|\n| 3|2020-03|2020|20202020| 03|\n| 1|2020-01|2020|20202020| 01|\n+---+-------+----+--------+-----+" } ], "execution_count": 11 @@ -457,7 +457,7 @@ " .option(\"dbtable\", dbtable) \\\r\n", " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", - " .option(\"schemaCheckEnabled\", False) \\\r\n", + " .option(\"columnsToWrite\", \"Id, Year\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -474,6 +474,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -482,13 +483,12 @@ "model_id": "2bc5b9342d5a4ae8ba5a4f030aedc00e" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) done", - "output_type": "stream" + "text": "MSSQL Connector write(append) done" } ], "execution_count": 12 @@ -518,6 +518,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -526,13 +527,12 @@ "model_id": "26d3b2fe1d1e488a9304c40be6991d14" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+---+----+--------+\n| Id|Year| Years|\n+---+----+--------+\n| 1|2020|20202020|\n| 2|2020|20202020|\n| 3|2020|20202020|\n+---+----+--------+", - "output_type": "stream" + "text": "+---+----+--------+\n| Id|Year| Years|\n+---+----+--------+\n| 1|2020|20202020|\n| 2|2020|20202020|\n| 3|2020|20202020|\n+---+----+--------+" } ], "execution_count": 13 @@ -563,7 +563,7 @@ " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", " .option(\"tableLock\",True) \\\r\n", - " .option(\"schemaCheckEnabled\", False)\\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -576,6 +576,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -584,13 +585,12 @@ "model_id": "451e02a8fa0d43cabe0032632b296546" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 15 @@ -615,6 +615,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -623,13 +624,12 @@ "model_id": "31ec9191e04341a28fb8654c4bf726da" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "read data from SQL server table \n+---+----+-----+--------------------------+--------------------------+\n|Id |Year|Month|ValidFrom |ValidTo |\n+---+----+-----+--------------------------+--------------------------+\n|1 |2020|01 |2021-10-25 06:52:27.622804|9999-12-31 23:59:59.999999|\n|2 |2020|02 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n|3 |2020|03 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n+---+----+-----+--------------------------+--------------------------+", - "output_type": "stream" + "text": "read data from SQL server table \n+---+----+-----+--------------------------+--------------------------+\n|Id |Year|Month|ValidFrom |ValidTo |\n+---+----+-----+--------------------------+--------------------------+\n|1 |2020|01 |2021-10-25 06:52:27.622804|9999-12-31 23:59:59.999999|\n|2 |2020|02 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n|3 |2020|03 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n+---+----+-----+--------------------------+--------------------------+" } ], "execution_count": 16 @@ -650,7 +650,7 @@ " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", " .option(\"tableLock\",True) \\\r\n", - " .option(\"schemaCheckEnabled\", False)\\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -662,6 +662,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -670,13 +671,12 @@ "model_id": "790a82624c2e4c6681debdc7db2efc1e" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 20 @@ -701,6 +701,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -709,13 +710,12 @@ "model_id": "87ba143688b246ffb9405b65d2cdb7a8" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "read data from SQL server table \n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|Date |Id |Year|Month|Years |ValidFrom |ValidTo |\n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|2020-01|1 |2020|01 |20202020|2021-10-25 07:02:34.68064|9999-12-31 23:59:59.999999|\n|2020-02|2 |2020|02 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n|2020-03|3 |2020|03 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n+-------+---+----+-----+--------+-------------------------+--------------------------+", - "output_type": "stream" + "text": "read data from SQL server table \n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|Date |Id |Year|Month|Years |ValidFrom |ValidTo |\n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|2020-01|1 |2020|01 |20202020|2021-10-25 07:02:34.68064|9999-12-31 23:59:59.999999|\n|2020-02|2 |2020|02 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n|2020-03|3 |2020|03 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n+-------+---+----+-----+--------+-------------------------+--------------------------+" } ], "execution_count": 21 @@ -746,7 +746,7 @@ " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", " .option(\"tableLock\",True) \\\r\n", - " .option(\"schemaCheckEnabled\", False)\\\r\n", + " .option(\"columnsToWrite\", \"PostID, PostTitle, PostBody\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -758,6 +758,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -766,13 +767,12 @@ "model_id": "1520e2d0d8294ce992442d4507a0479f" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 23 @@ -798,6 +798,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -806,13 +807,12 @@ "model_id": "a244c97f89df4bb38f6db0e77e3d64ed" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "read data from SQL server table \n+-----------------------------------------+------+------------+--------------------+\n|$node_id_E3C19E7D3E1648A2BD5364EF445EA866|PostID| PostTitle| PostBody|\n+-----------------------------------------+------+------------+--------------------+\n| {\"type\":\"node\",\"s...| 1| Intro|Hi There This is ABC|\n| {\"type\":\"node\",\"s...| 2| Intro| Hello Im PQR|\n| {\"type\":\"node\",\"s...| 3| Re: Intro| Hey PQR This is XYZ|\n| {\"type\":\"node\",\"s...| 4| Geography| Im George from USA|\n| {\"type\":\"node\",\"s...| 5|Re:Geography| Im Mary from OZ|\n+-----------------------------------------+------+------------+--------------------+\nonly showing top 5 rows", - "output_type": "stream" + "text": "read data from SQL server table \n+-----------------------------------------+------+------------+--------------------+\n|$node_id_E3C19E7D3E1648A2BD5364EF445EA866|PostID| PostTitle| PostBody|\n+-----------------------------------------+------+------------+--------------------+\n| {\"type\":\"node\",\"s...| 1| Intro|Hi There This is ABC|\n| {\"type\":\"node\",\"s...| 2| Intro| Hello Im PQR|\n| {\"type\":\"node\",\"s...| 3| Re: Intro| Hey PQR This is XYZ|\n| {\"type\":\"node\",\"s...| 4| Geography| Im George from USA|\n| {\"type\":\"node\",\"s...| 5|Re:Geography| Im Mary from OZ|\n+-----------------------------------------+------+------------+--------------------+\nonly showing top 5 rows" } ], "execution_count": 24