diff --git a/.gitignore b/.gitignore index 8c61629..46a1c42 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +bq-service-account.json + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/Dockerfile b/Dockerfile index 8785b17..dd9b2b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,9 @@ FROM ebmdatalab/datalab-jupyter:python3.8.1-2328e31e7391a127fe7184dcce38d581a17b # Set up jupyter environment ENV MAIN_PATH=/home/app/notebook +# Set path to BQ service account credentials +ENV EBMDATALAB_BQ_CREDENTIALS_PATH=/tmp/bq-service-account.json + # Install pip requirements COPY requirements.txt /tmp/ # Hack until this is fixed https://github.com/jazzband/pip-tools/issues/823 @@ -18,3 +21,7 @@ COPY config/kernel.json /tmp/kernel_with_custom_path/kernel.json RUN jupyter kernelspec install /tmp/kernel_with_custom_path/ --user --name="python3" CMD cd ${MAIN_PATH} && PYTHONPATH=${MAIN_PATH} jupyter lab --config=config/jupyter_notebook_config.py + +# Copy BQ service account credentials into container +# We work around the credentials not existing in CI with the glob +COPY bq-service-account.jso[n] /tmp/ diff --git a/notebooks/bnf_code_to_dmd.ipynb b/notebooks/bnf_code_to_dmd.ipynb index d54c953..95d006d 100644 --- a/notebooks/bnf_code_to_dmd.ipynb +++ b/notebooks/bnf_code_to_dmd.ipynb @@ -50,9 +50,18 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/root/.pyenv/versions/3.8.1/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3254: DtypeWarning: Columns (7) have mixed types.Specify dtype option on import or set low_memory=False.\n", + " if (await self.run_code(code, result, async_=asy)):\n" + ] + } + ], "source": [ "sql = \"\"\"\n", " SELECT\n", @@ -99,7 +108,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -216,7 +225,7 @@ "4 NaN NaN " ] }, - "execution_count": 19, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -248,31 +257,9 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 4, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Please visit this URL to authorize this application: https://accounts.google.com/o/oauth2/auth?response_type=code&client_id=725825577420-unm2gnkiprugilg743tkbig250f4sfsj.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fbigquery&state=TqQ4ZdHcluER8Hj6XqZcRU6rfoiI6Q&prompt=consent&access_type=offline\n" - ] - }, - { - "name": "stdin", - "output_type": "stream", - "text": [ - "Enter the authorization code: 4/1ARtbsJrjCR_Ou21T-ppSLpaAJVaDe4pSMPNqqLdoUztkLpgBRXASeeJm-dg\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Downloading: 100%|██████████| 27595/27595 [00:01<00:00, 19592.23rows/s]\n" - ] - } - ], + "outputs": [], "source": [ "sql = \"\"\"\n", "SELECT\n", @@ -295,7 +282,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -368,7 +355,7 @@ "4 21220000215 Zerocream 373162" ] }, - "execution_count": 28, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -386,7 +373,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -395,7 +382,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -531,7 +518,7 @@ "4 Irbesartan 300mg / Hydrochlorothiazide 12.5mg ... 19101 " ] }, - "execution_count": 30, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -549,7 +536,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -671,7 +658,7 @@ "108091 NaN Barkat gluten free wholemeal bread sliced 285 " ] }, - "execution_count": 31, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -697,7 +684,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -706,7 +693,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -1235,7 +1222,7 @@ "29727 Vitamin B compound strong tablets 1199121 " ] }, - "execution_count": 35, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -1246,7 +1233,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -1257,7 +1244,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -1448,7 +1435,7 @@ "Fortisip Compact liquid (8 flavours) 388180" ] }, - "execution_count": 42, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } diff --git a/requirements.txt b/requirements.txt index 4d3b25d..78b4d6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ cycler==0.10.0 # via matplotlib decorator==4.4.1 # via ipython, traitlets defusedxml==0.6.0 # via nbconvert descartes==1.1.0 # via ebmdatalab -ebmdatalab==0.0.21 # via -r requirements.in +ebmdatalab==0.0.29 # via -r requirements.in entrypoints==0.3 # via nbconvert fiona==1.8.13 # via geopandas geopandas==0.6.3 # via ebmdatalab