From 108a50ebc5bcf0c8f25b5f3ff6e24a32af7ca342 Mon Sep 17 00:00:00 2001 From: luxu1 Date: Tue, 14 Dec 2021 01:10:15 -0800 Subject: [PATCH] modify for new PR --- .../computed-col-test.ipynb | 274 +++++++++++++----- .../sql-table-generation.ipynb | 115 +++++++- 2 files changed, 298 insertions(+), 91 deletions(-) diff --git a/samples/computed-col-test-cases/computed-col-test.ipynb b/samples/computed-col-test-cases/computed-col-test.ipynb index 30a496e..f612041 100644 --- a/samples/computed-col-test-cases/computed-col-test.ipynb +++ b/samples/computed-col-test-cases/computed-col-test.ipynb @@ -50,19 +50,20 @@ }, "outputs": [ { + "output_type": "stream", "name": "stdout", - "text": "Starting Spark application\n", - "output_type": "stream" + "text": "Starting Spark application\n" }, { + "output_type": "display_data", "data": { "text/plain": "", "text/html": "\n
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
32application_1628887230618_0066pysparkidleLinkLink
" }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -71,15 +72,15 @@ "model_id": "57594e6996e94f2bb9181dee19ddaafa" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "SparkSession available as 'spark'.\n", - "output_type": "stream" + "text": "SparkSession available as 'spark'.\n" }, { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -88,8 +89,7 @@ "model_id": "49029d8d1d164fd0966752ecdf3aa4d5" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} } ], "execution_count": 2 @@ -124,6 +124,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -132,13 +133,12 @@ "model_id": "b0b0e4f5ed664b71aae1a27adc41ddac" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+---+----+-----+\n| Id|Year|Month|\n+---+----+-----+\n| 1|2020| 01|\n| 2|2020| 02|\n| 3|2020| 03|\n+---+----+-----+\n\n+------+------------+--------------------+\n|PostID| PostTitle| PostBody|\n+------+------------+--------------------+\n| 1| Intro|Hi There This is ABC|\n| 2| Intro| Hello Im PQR|\n| 3| Re: Intro| Hey PQR This is XYZ|\n| 4| Geography| Im George from USA|\n| 5|Re:Geography| Im Mary from OZ|\n| 6|Re:Geography| Im Peter from UK|\n+------+------------+--------------------+", - "output_type": "stream" + "text": "+---+----+-----+\n| Id|Year|Month|\n+---+----+-----+\n| 1|2020| 01|\n| 2|2020| 02|\n| 3|2020| 03|\n+---+----+-----+\n\n+------+------------+--------------------+\n|PostID| PostTitle| PostBody|\n+------+------------+--------------------+\n| 1| Intro|Hi There This is ABC|\n| 2| Intro| Hello Im PQR|\n| 3| Re: Intro| Hey PQR This is XYZ|\n| 4| Geography| Im George from USA|\n| 5|Re:Geography| Im Mary from OZ|\n| 6|Re:Geography| Im Peter from UK|\n+------+------------+--------------------+" } ], "execution_count": 5 @@ -168,7 +168,7 @@ " .option(\"dbtable\", dbtable) \\\r\n", " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", - " .option(\"schemaCheckEnabled\", False) \\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -185,6 +185,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -193,13 +194,12 @@ "model_id": "ef11011fce0a4d86b2ab67d3506ca6d0" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 6 @@ -229,6 +229,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -237,13 +238,12 @@ "model_id": "3d68cffa23f74740b5c234aec7036761" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+-------+---+----+-----+--------+\n| Date| Id|Year|Month| Years|\n+-------+---+----+-----+--------+\n|2020-02| 2|2020| 02|20202020|\n|2020-03| 3|2020| 03|20202020|\n|2020-01| 1|2020| 01|20202020|\n+-------+---+----+-----+--------+", - "output_type": "stream" + "text": "+-------+---+----+-----+--------+\n| Date| Id|Year|Month| Years|\n+-------+---+----+-----+--------+\n|2020-02| 2|2020| 02|20202020|\n|2020-03| 3|2020| 03|20202020|\n|2020-01| 1|2020| 01|20202020|\n+-------+---+----+-----+--------+" } ], "execution_count": 7 @@ -263,7 +263,7 @@ " .option(\"dbtable\", dbtable) \\\r\n", " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", - " .option(\"schemaCheckEnabled\", False) \\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -281,6 +281,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -289,13 +290,12 @@ "model_id": "d0f1fb3556824b68acf73d1ef0cc4d6b" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 8 @@ -325,6 +325,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -333,13 +334,12 @@ "model_id": "83f7f1511dd14c26a6b63053fac7a2d3" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+---+-------+----+-----+\n| Id| Date|Year|Month|\n+---+-------+----+-----+\n| 1|2020-01|2020| 01|\n| 2|2020-02|2020| 02|\n| 3|2020-03|2020| 03|\n+---+-------+----+-----+", - "output_type": "stream" + "text": "+---+-------+----+-----+\n| Id| Date|Year|Month|\n+---+-------+----+-----+\n| 1|2020-01|2020| 01|\n| 2|2020-02|2020| 02|\n| 3|2020-03|2020| 03|\n+---+-------+----+-----+" } ], "execution_count": 9 @@ -359,7 +359,7 @@ " .option(\"dbtable\", dbtable) \\\r\n", " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", - " .option(\"schemaCheckEnabled\", False) \\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -377,6 +377,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -385,13 +386,12 @@ "model_id": "880a7b25e6fc472dac6ca47f737ffdda" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) done", - "output_type": "stream" + "text": "MSSQL Connector write(append) done" } ], "execution_count": 10 @@ -422,6 +422,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -430,13 +431,12 @@ "model_id": "94f5c08789b849ffa6e7469d0ccb9e34" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+---+-------+----+--------+-----+\n| Id| Date|Year| Years|Month|\n+---+-------+----+--------+-----+\n| 2|2020-02|2020|20202020| 02|\n| 3|2020-03|2020|20202020| 03|\n| 1|2020-01|2020|20202020| 01|\n+---+-------+----+--------+-----+", - "output_type": "stream" + "text": "+---+-------+----+--------+-----+\n| Id| Date|Year| Years|Month|\n+---+-------+----+--------+-----+\n| 2|2020-02|2020|20202020| 02|\n| 3|2020-03|2020|20202020| 03|\n| 1|2020-01|2020|20202020| 01|\n+---+-------+----+--------+-----+" } ], "execution_count": 11 @@ -457,7 +457,7 @@ " .option(\"dbtable\", dbtable) \\\r\n", " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", - " .option(\"schemaCheckEnabled\", False) \\\r\n", + " .option(\"columnsToWrite\", \"Id, Year\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -474,6 +474,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -482,13 +483,12 @@ "model_id": "2bc5b9342d5a4ae8ba5a4f030aedc00e" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) done", - "output_type": "stream" + "text": "MSSQL Connector write(append) done" } ], "execution_count": 12 @@ -518,6 +518,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -526,13 +527,12 @@ "model_id": "26d3b2fe1d1e488a9304c40be6991d14" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "+---+----+--------+\n| Id|Year| Years|\n+---+----+--------+\n| 1|2020|20202020|\n| 2|2020|20202020|\n| 3|2020|20202020|\n+---+----+--------+", - "output_type": "stream" + "text": "+---+----+--------+\n| Id|Year| Years|\n+---+----+--------+\n| 1|2020|20202020|\n| 2|2020|20202020|\n| 3|2020|20202020|\n+---+----+--------+" } ], "execution_count": 13 @@ -563,7 +563,7 @@ " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", " .option(\"tableLock\",True) \\\r\n", - " .option(\"schemaCheckEnabled\", False)\\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -576,6 +576,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -584,13 +585,12 @@ "model_id": "451e02a8fa0d43cabe0032632b296546" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 15 @@ -615,6 +615,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -623,13 +624,12 @@ "model_id": "31ec9191e04341a28fb8654c4bf726da" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "read data from SQL server table \n+---+----+-----+--------------------------+--------------------------+\n|Id |Year|Month|ValidFrom |ValidTo |\n+---+----+-----+--------------------------+--------------------------+\n|1 |2020|01 |2021-10-25 06:52:27.622804|9999-12-31 23:59:59.999999|\n|2 |2020|02 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n|3 |2020|03 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n+---+----+-----+--------------------------+--------------------------+", - "output_type": "stream" + "text": "read data from SQL server table \n+---+----+-----+--------------------------+--------------------------+\n|Id |Year|Month|ValidFrom |ValidTo |\n+---+----+-----+--------------------------+--------------------------+\n|1 |2020|01 |2021-10-25 06:52:27.622804|9999-12-31 23:59:59.999999|\n|2 |2020|02 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n|3 |2020|03 |2021-10-25 06:52:27.602376|9999-12-31 23:59:59.999999|\n+---+----+-----+--------------------------+--------------------------+" } ], "execution_count": 16 @@ -650,7 +650,7 @@ " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", " .option(\"tableLock\",True) \\\r\n", - " .option(\"schemaCheckEnabled\", False)\\\r\n", + " .option(\"columnsToWrite\", \"Id, Year, Month\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -662,6 +662,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -670,13 +671,12 @@ "model_id": "790a82624c2e4c6681debdc7db2efc1e" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 20 @@ -701,6 +701,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -709,13 +710,12 @@ "model_id": "87ba143688b246ffb9405b65d2cdb7a8" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "read data from SQL server table \n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|Date |Id |Year|Month|Years |ValidFrom |ValidTo |\n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|2020-01|1 |2020|01 |20202020|2021-10-25 07:02:34.68064|9999-12-31 23:59:59.999999|\n|2020-02|2 |2020|02 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n|2020-03|3 |2020|03 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n+-------+---+----+-----+--------+-------------------------+--------------------------+", - "output_type": "stream" + "text": "read data from SQL server table \n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|Date |Id |Year|Month|Years |ValidFrom |ValidTo |\n+-------+---+----+-----+--------+-------------------------+--------------------------+\n|2020-01|1 |2020|01 |20202020|2021-10-25 07:02:34.68064|9999-12-31 23:59:59.999999|\n|2020-02|2 |2020|02 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n|2020-03|3 |2020|03 |20202020|2021-10-25 07:02:34.66426|9999-12-31 23:59:59.999999|\n+-------+---+----+-----+--------+-------------------------+--------------------------+" } ], "execution_count": 21 @@ -746,7 +746,7 @@ " .option(\"user\", user) \\\r\n", " .option(\"password\", password) \\\r\n", " .option(\"tableLock\",True) \\\r\n", - " .option(\"schemaCheckEnabled\", False)\\\r\n", + " .option(\"columnsToWrite\", \"PostID, PostTitle, PostBody\") \\\r\n", " .save()\r\n", "except ValueError as error :\r\n", " print(\"MSSQL Connector write failed\", error)\r\n", @@ -758,6 +758,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -766,13 +767,12 @@ "model_id": "1520e2d0d8294ce992442d4507a0479f" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "MSSQL Connector write(append) succeeded", - "output_type": "stream" + "text": "MSSQL Connector write(append) succeeded" } ], "execution_count": 23 @@ -798,6 +798,7 @@ }, "outputs": [ { + "output_type": "display_data", "data": { "text/plain": "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…", "application/vnd.jupyter.widget-view+json": { @@ -806,16 +807,135 @@ "model_id": "a244c97f89df4bb38f6db0e77e3d64ed" } }, - "metadata": {}, - "output_type": "display_data" + "metadata": {} }, { + "output_type": "stream", "name": "stdout", - "text": "read data from SQL server table \n+-----------------------------------------+------+------------+--------------------+\n|$node_id_E3C19E7D3E1648A2BD5364EF445EA866|PostID| PostTitle| PostBody|\n+-----------------------------------------+------+------------+--------------------+\n| {\"type\":\"node\",\"s...| 1| Intro|Hi There This is ABC|\n| {\"type\":\"node\",\"s...| 2| Intro| Hello Im PQR|\n| {\"type\":\"node\",\"s...| 3| Re: Intro| Hey PQR This is XYZ|\n| {\"type\":\"node\",\"s...| 4| Geography| Im George from USA|\n| {\"type\":\"node\",\"s...| 5|Re:Geography| Im Mary from OZ|\n+-----------------------------------------+------+------------+--------------------+\nonly showing top 5 rows", - "output_type": "stream" + "text": "read data from SQL server table \n+-----------------------------------------+------+------------+--------------------+\n|$node_id_E3C19E7D3E1648A2BD5364EF445EA866|PostID| PostTitle| PostBody|\n+-----------------------------------------+------+------------+--------------------+\n| {\"type\":\"node\",\"s...| 1| Intro|Hi There This is ABC|\n| {\"type\":\"node\",\"s...| 2| Intro| Hello Im PQR|\n| {\"type\":\"node\",\"s...| 3| Re: Intro| Hey PQR This is XYZ|\n| {\"type\":\"node\",\"s...| 4| Geography| Im George from USA|\n| {\"type\":\"node\",\"s...| 5|Re:Geography| Im Mary from OZ|\n+-----------------------------------------+------+------------+--------------------+\nonly showing top 5 rows" } ], "execution_count": 24 + }, + { + "cell_type": "markdown", + "source": [ + "### **4\\. Write to Tables with Identity columns**" + ], + "metadata": { + "azdata_cell_guid": "3fd6f1f2-cac4-42c1-a3c0-118de963d6ac" + }, + "attachments": {} + }, + { + "cell_type": "code", + "source": [ + "# auto increment columns: Write from Spark to SQL table using MSSQL Spark Connector\r\n", + "dbtable = \"test_identity\"\r\n", + "\r\n", + "try:\r\n", + " df.write \\\r\n", + " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", + " .mode(\"append\") \\\r\n", + " .option(\"truncate\", \"true\") \\\r\n", + " .option(\"url\", url) \\\r\n", + " .option(\"dbtable\", dbtable) \\\r\n", + " .option(\"user\", user) \\\r\n", + " .option(\"password\", password) \\\r\n", + " .option(\"tableLock\",True) \\\r\n", + " .option(\"columnsToWrite\", \"Year, Month\") \\\r\n", + " .save()\r\n", + "except ValueError as error :\r\n", + " print(\"MSSQL Connector write failed\", error)\r\n", + "\r\n", + "print(\"MSSQL Connector write(append) succeeded \")" + ], + "metadata": { + "azdata_cell_guid": "99bf84bc-c6bb-43a9-aa6b-9f3387cbcae1" + }, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "source": [ + "#Read from SQL table using MSSQ Connector\r\n", + "dbtable = \"test_identity\"\r\n", + "print(\"read data from SQL server table \")\r\n", + "jdbcDF = spark.read \\\r\n", + " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", + " .option(\"url\", url) \\\r\n", + " .option(\"dbtable\", dbtable) \\\r\n", + " .option(\"user\", user) \\\r\n", + " .option(\"password\", password).load()\r\n", + "\r\n", + "jdbcDF.show(5)" + ], + "metadata": { + "azdata_cell_guid": "d477c6a5-4e69-4006-8742-786b41fb4031" + }, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "markdown", + "source": [ + "### **5\\. Write to Table skipping some non-nullable columns**" + ], + "metadata": { + "azdata_cell_guid": "340e9ce8-eabb-4cd5-95e9-ff5b3e6b5b46" + }, + "attachments": {} + }, + { + "cell_type": "code", + "source": [ + "# skip non-nullable column: Write from Spark to SQL table using MSSQL Spark Connector\r\n", + "dbtable = \"test_skip_cols\"\r\n", + "\r\n", + "try:\r\n", + " df.write \\\r\n", + " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", + " .mode(\"append\") \\\r\n", + " .option(\"truncate\", \"true\") \\\r\n", + " .option(\"url\", url) \\\r\n", + " .option(\"dbtable\", dbtable) \\\r\n", + " .option(\"user\", user) \\\r\n", + " .option(\"password\", password) \\\r\n", + " .option(\"tableLock\",True) \\\r\n", + " .option(\"columnsToWrite\", \"Id, Year\") \\\r\n", + " .save()\r\n", + "except ValueError as error :\r\n", + " print(\"MSSQL Connector write failed\", error)\r\n", + "\r\n", + "print(\"MSSQL Connector write(append) succeeded \")" + ], + "metadata": { + "azdata_cell_guid": "98ee8100-f43f-468b-af97-9024144f002c" + }, + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "source": [ + "#Read from SQL table using MSSQ Connector\r\n", + "dbtable = \"test_skip_cols\"\r\n", + "print(\"read data from SQL server table \")\r\n", + "jdbcDF = spark.read \\\r\n", + " .format(\"com.microsoft.sqlserver.jdbc.spark\") \\\r\n", + " .option(\"url\", url) \\\r\n", + " .option(\"dbtable\", dbtable) \\\r\n", + " .option(\"user\", user) \\\r\n", + " .option(\"password\", password).load()\r\n", + "\r\n", + "jdbcDF.show(5)" + ], + "metadata": { + "azdata_cell_guid": "d687eb8f-435d-4ba9-b396-9e756185c4ca" + }, + "outputs": [], + "execution_count": null } ] } \ No newline at end of file diff --git a/samples/computed-col-test-cases/sql-table-generation.ipynb b/samples/computed-col-test-cases/sql-table-generation.ipynb index 052e9b9..44d3fec 100644 --- a/samples/computed-col-test-cases/sql-table-generation.ipynb +++ b/samples/computed-col-test-cases/sql-table-generation.ipynb @@ -43,7 +43,7 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.640" + "text/html": "Total execution time: 00:00:00.287" }, "metadata": {} } @@ -91,7 +91,7 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.108" + "text/html": "Total execution time: 00:00:00.022" }, "metadata": {} } @@ -142,12 +142,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.011" + "text/html": "Total execution time: 00:00:00.018" }, "metadata": {} } ], - "execution_count": 1 + "execution_count": 3 }, { "cell_type": "code", @@ -182,12 +182,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.010" + "text/html": "Total execution time: 00:00:00.017" }, "metadata": {} } ], - "execution_count": 2 + "execution_count": 4 }, { "cell_type": "code", @@ -224,12 +224,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.011" + "text/html": "Total execution time: 00:00:00.018" }, "metadata": {} } ], - "execution_count": 3 + "execution_count": 5 }, { "cell_type": "code", @@ -264,12 +264,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.012" + "text/html": "Total execution time: 00:00:00.015" }, "metadata": {} } ], - "execution_count": 4 + "execution_count": 6 }, { "cell_type": "markdown", @@ -310,12 +310,12 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.015" + "text/html": "Total execution time: 00:00:00.022" }, "metadata": {} } ], - "execution_count": 13 + "execution_count": 7 }, { "cell_type": "code", @@ -359,7 +359,7 @@ "metadata": {} } ], - "execution_count": 27 + "execution_count": 8 }, { "cell_type": "code", @@ -440,7 +440,94 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.027" + "text/html": "Total execution time: 00:00:00.046" + }, + "metadata": {} + } + ], + "execution_count": 9 + }, + { + "cell_type": "markdown", + "source": [ + "### **4\\. Table with Identity column**" + ], + "metadata": { + "azdata_cell_guid": "8af5cf11-1bec-4049-a002-0678e0075afb" + }, + "attachments": {} + }, + { + "cell_type": "code", + "source": [ + "USE connector_test_db\r\n", + "\r\n", + "CREATE TABLE test_identity\r\n", + "(\r\n", + " Id bigint IDENTITY(1,1),\r\n", + " Year nvarchar(4),\r\n", + " Month nvarchar(2),\r\n", + " Years AS REPLICATE(Year, 2)\r\n", + ")" + ], + "metadata": { + "azdata_cell_guid": "66493ab5-2e0a-4249-b214-49f027008723" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/html": "Total execution time: 00:00:00.043" + }, + "metadata": {} + } + ], + "execution_count": 12 + }, + { + "cell_type": "markdown", + "source": [ + "### **5\\. Insert to table skipping some columns**" + ], + "metadata": { + "azdata_cell_guid": "6d24c74f-e28b-4bc1-9d86-2ba0bcb37f0b" + }, + "attachments": {} + }, + { + "cell_type": "code", + "source": [ + "USE connector_test_db\r\n", + "\r\n", + "CREATE TABLE test_skip_cols\r\n", + "(\r\n", + " Id bigint,\r\n", + " Year nvarchar(4),\r\n", + " Month nvarchar(2)\r\n", + ")" + ], + "metadata": { + "azdata_cell_guid": "7496e3c4-d5d0-46cc-b8c4-2cfaa40f01ae" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/html": "Total execution time: 00:00:00.019" }, "metadata": {} }