Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(ingest/setup): Fix for feast and abs source setup #10951

Merged
merged 3 commits into from
Jul 19, 2024
Merged

fix(ingest/setup): Fix for feast and abs source setup #10951

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5065224
Fixing setup.py
treff7es Jul 19, 2024
7a09e2d
Adding missing dependencites to abs source
treff7es Jul 19, 2024
2b71c62
Merge branch 'master' into feast_fix
treff7es Jul 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions metadata-ingestion/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,12 @@
"azure-identity>=1.14.0",
"azure-storage-blob>=12.19.0",
"azure-storage-file-datalake>=12.14.0",
"more-itertools>=8.12.0",
"pyarrow>=6.0.1",
"smart-open[azure]>=5.2.1",
"tableschema>=1.20.2",
"ujson>=5.2.0",
*path_spec_common,
}

data_lake_profiling = {
Expand Down Expand Up @@ -352,6 +358,10 @@
"feast>=0.34.0,<1",
"flask-openid>=1.3.0",
"dask[dataframe]<2024.7.0",
# We were seeing an error like this `numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject`
# with numpy 2.0. This likely indicates a mismatch between scikit-learn and numpy versions.
# https://stackoverflow.com/questions/40845304/runtimewarning-numpy-dtype-size-changed-may-indicate-binary-incompatibility
"numpy<2",
},
"grafana": {"requests"},
"glue": aws_common,
Expand Down Expand Up @@ -415,7 +425,7 @@
| {"cachetools"},
"s3": {*s3_base, *data_lake_profiling},
"gcs": {*s3_base, *data_lake_profiling},
"abs": {*abs_base},
"abs": {*abs_base, *data_lake_profiling},
"sagemaker": aws_common,
"salesforce": {"simple-salesforce"},
"snowflake": snowflake_common | usage_common | sqlglot_lib,
Expand Down Expand Up @@ -539,6 +549,7 @@
*list(
dependency
for plugin in [
"abs",
"athena",
"bigquery",
"clickhouse",
Expand Down Expand Up @@ -627,6 +638,7 @@
entry_points = {
"console_scripts": ["datahub = datahub.entrypoints:main"],
"datahub.ingestion.source.plugins": [
"abs = datahub.ingestion.source.abs.source:ABSSource",
"csv-enricher = datahub.ingestion.source.csv_enricher:CSVEnricherSource",
"file = datahub.ingestion.source.file:GenericFileSource",
"datahub = datahub.ingestion.source.datahub.datahub_source:DataHubSource",
Expand Down Expand Up @@ -695,7 +707,6 @@
"demo-data = datahub.ingestion.source.demo_data.DemoDataSource",
"unity-catalog = datahub.ingestion.source.unity.source:UnityCatalogSource",
"gcs = datahub.ingestion.source.gcs.gcs_source:GCSSource",
"abs = datahub.ingestion.source.abs.source:ABSSource",
"sql-queries = datahub.ingestion.source.sql_queries:SqlQueriesSource",
"fivetran = datahub.ingestion.source.fivetran.fivetran:FivetranSource",
"qlik-sense = datahub.ingestion.source.qlik_sense.qlik_sense:QlikSenseSource",
Expand Down
Loading