From f71ae556f1bbe44b1dff4f1ec502ac0ba45371f5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 9 Jan 2024 16:13:14 +0000 Subject: [PATCH] auto updating docs --- docs/build/doctrees/environment.pickle | Bin 69325 -> 68839 bytes .../nesta_ds_utils/loading_saving/S3.html | 67 +++++++----------- 2 files changed, 24 insertions(+), 43 deletions(-) diff --git a/docs/build/doctrees/environment.pickle b/docs/build/doctrees/environment.pickle index 1a8654a93cb1103aaea22dfbd9e1d2d832b10653..013d1460839e6cc9ecfbf4922bd33a54004c07a7 100644 GIT binary patch delta 2796 zcmbtWeNa@_73VH2l0}JpiYU75?f}aI8ciL9s)&f2tc$3KqL7yd`xf`HyKlSuHiANk z){m~UTJ&r_I%=9UX~uLqsc@OjBpnAbwWi55Gc6w?Vw<52Njj#TX-93FiD}Qh3rk3y z{?px=chB#4?mh3^-#zcV_u-|4Pk)f0eNOxQwd6kS%Le$=klUE#bh#x#@dYB`B(J0% zQvB^`YId)UgcC)#D!TlUu+FLYJ))|*yls(iidzvKXbA!F8BQe58d{gvA$4G!R<|U2 z=~AOEzGqw$7q@NmL|aq-p3Cdxn{()^_3}sz?VZ>u|2~HHrCH?R7;1d?fjkyN2a+Dh z6LYBlJ^6MFH9cM?-2}ZrMj}^Vy#@l*6mVmGH6I2ni)kP-ij0@sd4IS89YW zOX}f$ixC2)EnGs`5+4`G<&&$K#iWyWGK}%^`8>0j zMDh=rb%R+yWfoILew$fL75O?^!El1Av<4#D?nt3LP8(Nc68PkxiJo8mEHhGR-kQ5n zBmePxQKNX>{iw0F>+-9x$C*Z7tgsPo-2JvfP`MS`y{KF| zOmDQp;x(zzvpSvb+~g++(&$*_D+Ja&t)Qk^@I{psw4w=|Tdu$@F^v{h|AIhgL+<&S z0g(5a=*^l=f=23VEmjR1Y2eA95C~TJ@C2HnYKw`c*q^gQ&{JjCwHqOd%Ac>;5Y zWx&vl6z)9!NED9Jjo9{3h_W8E_?{J zI?<_M=egIp1Fu^T*WxvDxDKz=hi&*AJ$7Ut_N>!K>hb!~5dpi_rmo$1g|1zg!?VS|eP|IUd4q|O-)-k4?=>-U?IWDzZ6`+V z5IM;^P>lSj2Bvz_d-{#s$2(EX_uFNhWWvcY@&z3yd4GzL=N531x2hQVQZpxc*NTzp zJhNah7*2F~#5M_EXvqN|Tzc*u&4mhRJbDl&j$VY##|-OB+gt&`eTOAejR<(x;9>-6|zt9Lwnbs?X?CJAj0r=ThVd&Md(a`BvFhG9;e6Lpwr z;Mip?RZl-ig{g~6YVp2mr%x)xmF7}G@QNNu5cC^1=ndN>rCLzeKujap$^~P=yiDms_1s>3s8&# zz0{%lMYij7vtIEdDJeeD>k!eUD*B+0eH7@CbYNaimRF$E00*X*LGsVj=sz!B(9-tT zvr`s-Gd=K3XTz$#h44cE8o1V<^?zq)c}R8+<&gJ(%!zr}pX(L>~6XIy>6{tmn~v^3``v44Pbg7_&A z*3Hc05yoL^DBa9@+c(y8YEkTVOo#}6MdZG#$t5FH|A%vnvRLB?FKUxi0gFZwe4-!s zI85KW)0DWB^^dTQ7hNl>LVRB!6Ab6W-_#XQc_@jt{Czla0ef2o!lv`xIq=td9Tomn NnMo#Jd?kgH{0G>Y+yDRo delta 2673 zcmZWre^k@g73V=B77OwlE9hd9BWNI1v^y-!A4R~_n1Fy*3mCqTZy?W*Z>q>TEYo?b*&`U;D$;g0#ou>9*Rng8YzntKIC{qdTX6wC=3O*4_64Vb&z) zz5BWMzWcrJ^X|Lf{Ogyh%fC{o52~MeC3!&og0@TT@;Jm`BphG6DiTg|O1^eMmR+9K zNO*3GTNFG|(dlxFcFF6Hgy;B$j;PKf%DW}sF4R5f5M)tywNJaKn*%PlBN9&NaP7eb zf?F0`zDPL5EeQ@>YM+kK7iey)QnWgf&@~bt7q{`zNpbftULF1~{V0ZB%iKz@#n9_F z-=tS#=nZ`dos6OLmpkZK4856hg^tdmbGh_N4865lPd}bTyBg?3!7PkiqT?|%w9-PS zW|8YJ@aLtaWCRu#ECOAA4!H)U`77a*{Kar2KbMR_f4+%~!u9+TGLAmS(DO{eMJO-S zLEo}ExQ=t;@>-}i=;6Tft?(y<4sI`h0-B2SaCCDPgbeGU+NgtDhV3x2TnC?TGr{S? z_3#ABs-gxm0YXs_BX2Qi;Z9K(iBIOD%*8U;S7jP~0RTa~W`Btr^sUUfHtFM^H#pM%SMvuxPUv`OJcE%FGb9r9sz< z3~+8ZORg$8>~ z2M8Hcj@mNgaeA@(JOQ0s!oyhrpVpY+rdzMrYB7zQ?lk4Yx|a$3qhb+wtyb_pqQ;|p z63$iXm1B=rVARSq_~-U7!HkqMp zgC3R%1CU*wuG|uk4<D8uAxQE%g91_4O!xO z0U=`5V@cqpgA=P8w6UT=#3}||Sh?WDY6Wdma6{U#gx%#vERb(hST@<}xHypqY>d#0 z2n2cs9`oicVF8$zXD)j6;vn*1=yer=fUypH4P7i&9}n^r`bH6p!VoL-KKjSRC}Mf_ z(f95_H|{~--GkV3SrnE>A5UNwu?!fRNa4nxJrIRc%G#gYA!I};_1_|MquZp{;Nd`U zF3SZ|pH%)3z=IxBE_G-L_JuEYAI@fpp((oGnLUm}chF0w;C68H26hA?zPAuRfDqqV zh|?e9?1wn{AM+)&kAx?Y@I{>-N>+2>WU<$_-g06}?E_rNyH8BH)y|c?5yg~GRd6NmNi)g>>q}a=-j%V{o6C)iIXYvs z_9k&9?^7}5bBSEZTUJat(7=_vd&QI*=soz?_=+{K<4{{lt0;K|k3;Zl@Yf2jAL>^O zKkZkORyf+7liKQa7&nSixgZObKA~NNffFSOsDRJA*P_eP!^3d+NJfFfX~%b=V3Z|0 zdu0|h2b@mPSEzCLTpglsjTTpIs;#nV;cicL0zMI{dJEvhKn7GFGb$hUew+3Xdr-6B z7unOQc|K)`mYI@Z)$?U=@MIRe|6Go;@#OPrFrJ+UBmD*lpIXgh4WIh?Lbg1Xq)Ed> zz8`#)@aq*U$6}u-2YepQ_jhD;`lNQdQxwoD`U{dYvmf~O#r*7L_1YPuwlQW^-alQK z0>iImU^c}t^y((1Z9tiqkZSn>EFOGFqqUfNBH{V=RwQPc6RDX-=rF6JHI^zc4dyy( zEJ1WiwYyz@`F^jxmg78xU-sQ^e8$p`#!S1%YZrVzq4R!^GnNbJkpq1va^Qo(#BJ|b zMiD0kyzJkIeKwhK%N^9G@MA+Q3*>N4CwkYKO3*m}|Bbk#sMVT5tQtre%2VlxwFrzT ztRU7$!8VkuDkoMOv=3=iTZpv@o*r7DY9ZFQ;rBy1T^EV2uw@=?z7LaMkxCGi4yG@rZs|utVFE3zYf`f6G&|E9{4+ z3mRqTNA1bk?4E{`TSb`_vS{}TKAE+$>t8g?$zaztoFoQipU@%;Zg+%^Db-(oH;0TW cRsW$`iR{Yx!x}jBSource code for nesta_ds_utils.loading_saving.S3
import boto3 from fnmatch import fnmatch import pandas as pd -import geopandas as gpd import numpy as np import pyarrow as pa import pyarrow.parquet as pq @@ -49,6 +48,14 @@

Source code for nesta_ds_utils.loading_saving.S3

import warnings from nesta_ds_utils.loading_saving import file_ops +from nesta_ds_utils.loading_saving.gis_interface import _gis_enabled + +if _gis_enabled: + from nesta_ds_utils.loading_saving.gis_interface import ( + _gdf_to_fileobj, + _fileobj_to_gdf, + ) + def get_bucket_filenames_s3(bucket_name: str, dir_name: str = "") -> List[str]: """Get a list of all files in bucket directory. @@ -95,27 +102,6 @@

Source code for nesta_ds_utils.loading_saving.S3

return buffer -def _gdf_to_fileobj(df_data: gpd.GeoDataFrame, path_to: str, **kwargs) -> io.BytesIO: - """Convert GeoDataFrame into bytes file object. - - Args: - df_data (gpd.DataFrame): Dataframe to convert. - path_to (str): Saving file name. - - Returns: - io.BytesIO: Bytes file object. - """ - buffer = io.BytesIO() - if fnmatch(path_to, "*.geojson"): - df_data.to_file(buffer, driver="GeoJSON", **kwargs) - else: - raise NotImplementedError( - "Uploading geodataframe currently supported only for 'geojson'." - ) - buffer.seek(0) - return buffer - - def _dict_to_fileobj(dict_data: dict, path_to: str, **kwargs) -> io.BytesIO: """Convert dictionary into bytes file object. @@ -266,10 +252,16 @@

Source code for nesta_ds_utils.loading_saving.S3

kwargs_writing (dict, optional): Dictionary of kwargs for writing data. """ - if isinstance(obj, gpd.base.GeoPandasBase): - obj = _gdf_to_fileobj(obj, path_to, **kwargs_writing) - elif isinstance(obj, pd.DataFrame): - obj = _df_to_fileobj(obj, path_to, **kwargs_writing) + if isinstance(obj, pd.DataFrame): + if type(obj).__name__ == "GeoDataFrame": + if _gis_enabled: + obj = _gdf_to_fileobj(obj, path_to, **kwargs_writing) + else: + raise ModuleNotFoundError( + "Please install 'gis' extra from nesta_ds_utils or 'geopandas' to upload geodataframes." + ) + else: + obj = _df_to_fileobj(obj, path_to, **kwargs_writing) elif isinstance(obj, dict): obj = _dict_to_fileobj(obj, path_to, **kwargs_writing) elif isinstance(obj, list): @@ -309,22 +301,6 @@

Source code for nesta_ds_utils.loading_saving.S3

return pd.read_excel(fileobj, **kwargs) -def _fileobj_to_gdf(fileobj: io.BytesIO, path_from: str, **kwargs) -> pd.DataFrame: - """Convert bytes file object into geodataframe. - - Args: - fileobj (io.BytesIO): Bytes file object. - path_from (str): Path of loaded data. - - Returns: - gpd.DataFrame: Data as geodataframe. - """ - if fnmatch(path_from, "*.geojson"): - return gpd.GeoDataFrame.from_features( - json.loads(fileobj.getvalue().decode())["features"] - ) - - def _fileobj_to_dict(fileobj: io.BytesIO, path_from: str, **kwargs) -> dict: """Convert bytes file object into dictionary. @@ -436,7 +412,12 @@

Source code for nesta_ds_utils.loading_saving.S3

) elif download_as == "geodf": if path_from.endswith(tuple([".geojson"])): - return _fileobj_to_gdf(fileobj, path_from, **kwargs_reading) + if _gis_enabled: + return _fileobj_to_gdf(fileobj, path_from, **kwargs_reading) + else: + raise ModuleNotFoundError( + "Please install 'gis' extra from nesta_ds_utils or 'geopandas' to download geodataframes." + ) else: raise NotImplementedError( "Download as geodataframe currently supported only " "for 'geojson'."