Skip to content
This repository has been archived by the owner on Jan 2, 2024. It is now read-only.

Commit

Permalink
Doc/ref man fixes (#341)
Browse files Browse the repository at this point in the history
* mypy fixes

* add missing status to the ref man

* formatting ref man

* hide DefaultJSONEncoder and DefaultJSONDecoder from doc

* hide DefaultJSONEncoder and DefaultJSONDecoder from doc

* expose Operator and JoinOperator at core.data level

* expose all exceptions at core.exceptions level

* hide Subscriber object

* Expose Scope and frequency at the right level

* Change version to 2.0.2

* Update src/taipy/core/common/_utils.py

Co-authored-by: Toan Quach <93168955+toan-quach@users.noreply.github.com>

Co-authored-by: Toan Quach <93168955+toan-quach@users.noreply.github.com>
  • Loading branch information
jrobinAV and toan-quach authored Oct 17, 2022
1 parent c458fff commit 36f5d60
Show file tree
Hide file tree
Showing 22 changed files with 58 additions and 52 deletions.
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
test_suite="tests",
tests_require=test_requirements,
url="https://github.com/avaiga/taipy-core",
version="2.0.1",
version="2.0.2",
zip_safe=False,
extras_require=extras_require,
)
4 changes: 2 additions & 2 deletions src/taipy/core/common/_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def _fct_to_dict(obj):
params = []
callback = obj

if isinstance(obj, Subscriber):
if isinstance(obj, _Subscriber):
callback = obj.callback
params = obj.params

Expand All @@ -52,4 +52,4 @@ def _fcts_to_dict(objs):
return [d for obj in objs if (d := _fct_to_dict(obj)) is not None]


Subscriber = namedtuple("Subscriber", "callback params")
_Subscriber = namedtuple("_Subscriber", "callback params")
3 changes: 3 additions & 0 deletions src/taipy/core/config/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@

from taipy.config.checker._checker import _Checker
from taipy.config.config import Config
from taipy.config.common.frequency import Frequency
from taipy.config.common.scope import Scope
from taipy.config.global_app.global_app_config import GlobalAppConfig

from .checkers._data_node_config_checker import _DataNodeConfigChecker
from .checkers._job_config_checker import _JobConfigChecker
Expand Down
2 changes: 1 addition & 1 deletion src/taipy/core/config/pipeline_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ def _to_dict(self):
@classmethod
def _from_dict(cls, as_dict: Dict[str, Any], id: str, config: Optional[_Config]):
as_dict.pop(cls._ID_KEY, id)
t_configs = config._sections[TaskConfig.name]
t_configs = config._sections[TaskConfig.name] # type: ignore
tasks = []
if tasks_ids := as_dict.pop(cls._TASK_KEY, None):
tasks = [t_configs[task_id] for task_id in tasks_ids if task_id in t_configs]
Expand Down
4 changes: 2 additions & 2 deletions src/taipy/core/config/scenario_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,9 @@ def _to_dict(self):
}

@classmethod
def _from_dict(cls, as_dict: Dict[str, Any], id: str, config: Optional[_Config]):
def _from_dict(cls, as_dict: Dict[str, Any], id: str, config: Optional[_Config]): # type: ignore
as_dict.pop(cls._ID_KEY, id)
p_configs = config._sections[PipelineConfig.name]
p_configs = config._sections[PipelineConfig.name] # type: ignore
pipelines = []
if pipeline_ids := as_dict.pop(cls._PIPELINE_KEY, None):
pipelines = [p_configs[p_id] for p_id in pipeline_ids if p_id in p_configs]
Expand Down
8 changes: 3 additions & 5 deletions src/taipy/core/config/task_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,9 @@ class TaskConfig(Section):
Attributes:
id (str): Identifier of the task config. Must be a valid Python variable name.
inputs (Union[DataNodeConfig, List[DataNodeConfig]]): The optional list of `DataNodeConfig^` inputs. The
default
value is [].
default value is [].
outputs (Union[DataNodeConfig, List[DataNodeConfig]]): The optional list of `DataNodeConfig^` outputs. The
default
value is [].
default value is [].
function (Callable): User function taking as inputs some parameters compatible with the exposed types
(exposed_type field) of the input data nodes and returning results compatible with the exposed types
(exposed_type field) of the outputs list. The default value is None.
Expand Down Expand Up @@ -101,7 +99,7 @@ def _to_dict(self):
def _from_dict(cls, as_dict: Dict[str, Any], id: str, config: Optional[_Config]):
as_dict.pop(cls._ID_KEY, id)
funct = as_dict.pop(cls._FUNCTION, None)
dn_configs = config._sections[DataNodeConfig.name]
dn_configs = config._sections[DataNodeConfig.name] # type: ignore
inputs = []
if inputs_as_str := as_dict.pop(cls._INPUT_KEY, None):
inputs = [dn_configs[dn_id] for dn_id in inputs_as_str if dn_id in dn_configs]
Expand Down
1 change: 1 addition & 0 deletions src/taipy/core/data/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from .generic import GenericDataNode
from .in_memory import InMemoryDataNode
from .json import JSONDataNode
from .operator import JoinOperator, Operator
from .pickle import PickleDataNode
from .sql import SQLDataNode
from .sql_table import SQLTableDataNode
5 changes: 2 additions & 3 deletions src/taipy/core/data/abstract_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,17 @@

import pandas as pd
from sqlalchemy import create_engine, text

from taipy.config.common.scope import Scope

from .data_node import DataNode
from ..common.alias import DataNodeId, JobId
from ..exceptions.exceptions import InvalidExposedType, MissingRequiredProperty, UnknownDatabaseEngine
from .data_node import DataNode


class AbstractSQLDataNode(DataNode):
"""Abstract base class for data node implementations (SQLDataNode and SQLTableDataNode) that use SQL."""

__STORAGE_TYPE = None
__STORAGE_TYPE = "NOT_IMPLEMENTED"
__EXPOSED_TYPE_NUMPY = "numpy"
__EXPOSED_TYPE_PANDAS = "pandas"
__VALID_STRING_EXPOSED_TYPES = [__EXPOSED_TYPE_PANDAS, __EXPOSED_TYPE_NUMPY]
Expand Down
8 changes: 4 additions & 4 deletions src/taipy/core/data/json.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,8 @@ def __init__(
edit_in_progress,
**properties,
)
self._decoder = self._properties.get(self._DECODER_KEY, DefaultJSONDecoder)
self._encoder = self._properties.get(self._ENCODER_KEY, DefaultJSONEncoder)
self._decoder = self._properties.get(self._DECODER_KEY, _DefaultJSONDecoder)
self._encoder = self._properties.get(self._ENCODER_KEY, _DefaultJSONEncoder)

if not self._last_edit_date and isfile(self._path): # type: ignore
self.unlock_edit()
Expand Down Expand Up @@ -143,7 +143,7 @@ def _write(self, data: Any):
json.dump(data, f, indent=4, cls=self._encoder)


class DefaultJSONEncoder(json.JSONEncoder):
class _DefaultJSONEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, Enum):
return o.value
Expand All @@ -154,5 +154,5 @@ def default(self, o):
return super().default(o)


class DefaultJSONDecoder(json.JSONDecoder):
class _DefaultJSONDecoder(json.JSONDecoder):
pass
2 changes: 2 additions & 0 deletions src/taipy/core/exceptions/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.

from .exceptions import *
3 changes: 3 additions & 0 deletions src/taipy/core/job/status.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ class Status(_ReprEnum):
- `COMPLETED`: A `COMPLETED` job has successfully been executed.
- `SKIPPED`: A `SKIPPED` job has not been executed because its outputs were already computed.
- `ABANDONED`: An `ABANDONED` job has not been executed because it depends on a job that could not complete (
cancelled, failed, or abandoned).
"""

SUBMITTED = 1
Expand Down
10 changes: 5 additions & 5 deletions src/taipy/core/pipeline/_pipeline_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

from taipy.config.common.scope import Scope

from ._pipeline_repository_factory import _PipelineRepositoryFactory
from .pipeline import Pipeline
from .._manager._manager import _Manager
from ..common._entity_ids import _EntityIds
from ..common.alias import PipelineId, ScenarioId
Expand All @@ -22,8 +24,6 @@
from ..job._job_manager_factory import _JobManagerFactory
from ..job.job import Job
from ..task._task_manager_factory import _TaskManagerFactory
from ._pipeline_repository_factory import _PipelineRepositoryFactory
from .pipeline import Pipeline


class _PipelineManager(_Manager[Pipeline]):
Expand Down Expand Up @@ -73,18 +73,18 @@ def __remove_subscriber(cls, callback, params, pipeline):

@classmethod
def _get_or_create(cls, pipeline_config: PipelineConfig, scenario_id: Optional[ScenarioId] = None) -> Pipeline:
pipeline_id = Pipeline._new_id(pipeline_config.id)
pipeline_id = Pipeline._new_id(str(pipeline_config.id))

task_manager = _TaskManagerFactory._build_manager()
tasks = task_manager._bulk_get_or_create(pipeline_config.task_configs, scenario_id, pipeline_id)

scope = min(task.scope for task in tasks) if len(tasks) != 0 else Scope.GLOBAL
parent_id = scenario_id if scope == Scope.SCENARIO else pipeline_id if scope == Scope.PIPELINE else None

if pipelines_from_parent := cls._repository._get_by_config_and_parent_id(pipeline_config.id, parent_id): # type: ignore
if pipelines_from_parent := cls._repository._get_by_config_and_parent_id(str(pipeline_config.id), parent_id):
return pipelines_from_parent

pipeline = Pipeline(pipeline_config.id, dict(**pipeline_config._properties), tasks, pipeline_id, parent_id)
pipeline = Pipeline(str(pipeline_config.id), dict(**pipeline_config._properties), tasks, pipeline_id, parent_id)
cls._set(pipeline)
return pipeline

Expand Down
4 changes: 2 additions & 2 deletions src/taipy/core/pipeline/_pipeline_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from .._repository._repository import _AbstractRepository
from .._repository._repository_adapter import _RepositoryAdapter
from ..common import _utils
from ..common._utils import Subscriber
from ..common._utils import _Subscriber
from ..exceptions.exceptions import NonExistingPipeline, NonExistingTask
from ..task.task import Task
from ._pipeline_model import _PipelineModel
Expand Down Expand Up @@ -59,7 +59,7 @@ def _from_model(self, model: _PipelineModel) -> Pipeline:
model.id,
model.parent_id,
[
Subscriber(_utils._load_fct(it["fct_module"], it["fct_name"]), it["fct_params"])
_Subscriber(_utils._load_fct(it["fct_module"], it["fct_name"]), it["fct_params"])
for it in model.subscribers
], # type: ignore
)
Expand Down
8 changes: 4 additions & 4 deletions src/taipy/core/pipeline/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from ..common._listattributes import _ListAttributes
from ..common._properties import _Properties
from ..common._reload import _reload, _self_reload, _self_setter
from ..common._utils import Subscriber
from ..common._utils import _Subscriber
from ..common.alias import PipelineId, TaskId
from ..data.data_node import DataNode
from ..exceptions.exceptions import NonExistingTask
Expand Down Expand Up @@ -54,7 +54,7 @@ def __init__(
tasks: Union[List[TaskId], List[Task], List[Union[TaskId, Task]]],
pipeline_id: PipelineId = None,
parent_id: Optional[str] = None,
subscribers: List[Subscriber] = None,
subscribers: List[_Subscriber] = None,
):
self.config_id = _validate_id(config_id)
self.id: PipelineId = pipeline_id or self._new_id(self.config_id)
Expand Down Expand Up @@ -181,11 +181,11 @@ def subscribers(self, val):

def _add_subscriber(self, callback: Callable, params: Optional[List[Any]] = None):
params = [] if params is None else params
self._subscribers.append(Subscriber(callback=callback, params=params))
self._subscribers.append(_Subscriber(callback=callback, params=params))

def _remove_subscriber(self, callback: Callable, params: Optional[List[Any]] = None):
if params is not None:
self._subscribers.remove(Subscriber(callback, params))
self._subscribers.remove(_Subscriber(callback, params))
else:
elem = [x for x in self._subscribers if x.callback == callback]
if not elem:
Expand Down
8 changes: 4 additions & 4 deletions src/taipy/core/scenario/_scenario_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@

from taipy.config.config import Config

from ._scenario_repository_factory import _ScenarioRepositoryFactory
from .scenario import Scenario
from .._manager._manager import _Manager
from ..common._entity_ids import _EntityIds
from ..common.alias import ScenarioId
Expand All @@ -34,8 +36,6 @@
from ..job._job_manager_factory import _JobManagerFactory
from ..job.job import Job
from ..pipeline._pipeline_manager_factory import _PipelineManagerFactory
from ._scenario_repository_factory import _ScenarioRepositoryFactory
from .scenario import Scenario


class _ScenarioManager(_Manager[Scenario]):
Expand Down Expand Up @@ -91,7 +91,7 @@ def _create(
creation_date: datetime.datetime = None,
name: str = None,
) -> Scenario:
scenario_id = Scenario._new_id(config.id)
scenario_id = Scenario._new_id(str(config.id))
pipelines = [
_PipelineManagerFactory._build_manager()._get_or_create(p_config, scenario_id)
for p_config in config.pipeline_configs
Expand All @@ -106,7 +106,7 @@ def _create(
if name:
props["name"] = name
scenario = Scenario(
config.id,
str(config.id),
pipelines, # type: ignore
props,
scenario_id,
Expand Down
4 changes: 2 additions & 2 deletions src/taipy/core/scenario/_scenario_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from .._repository._repository import _AbstractRepository
from .._repository._repository_adapter import _RepositoryAdapter
from ..common import _utils
from ..common._utils import Subscriber
from ..common._utils import _Subscriber
from ..common.alias import CycleId, PipelineId
from ..cycle._cycle_manager_factory import _CycleManagerFactory
from ..cycle.cycle import Cycle
Expand Down Expand Up @@ -57,7 +57,7 @@ def _from_model(self, model: _ScenarioModel) -> Scenario:
tags=set(model.tags),
cycle=self.__to_cycle(model.cycle),
subscribers=[
Subscriber(_utils._load_fct(it["fct_module"], it["fct_name"]), it["fct_params"])
_Subscriber(_utils._load_fct(it["fct_module"], it["fct_name"]), it["fct_params"])
for it in model.subscribers
],
)
Expand Down
8 changes: 4 additions & 4 deletions src/taipy/core/scenario/scenario.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from ..common._listattributes import _ListAttributes
from ..common._properties import _Properties
from ..common._reload import _reload, _self_reload, _self_setter
from ..common._utils import Subscriber
from ..common._utils import _Subscriber
from ..common.alias import PipelineId, ScenarioId
from ..cycle.cycle import Cycle
from ..data.data_node import DataNode
Expand Down Expand Up @@ -63,7 +63,7 @@ def __init__(
creation_date=None,
is_primary: bool = False,
cycle: Cycle = None,
subscribers: List[Subscriber] = None,
subscribers: List[_Subscriber] = None,
tags: Set[str] = None,
):
self.config_id = _validate_id(config_id)
Expand Down Expand Up @@ -198,7 +198,7 @@ def __getattr__(self, attribute_name):

def _add_subscriber(self, callback: Callable, params: Optional[List[Any]] = None):
params = [] if params is None else params
self._subscribers.append(Subscriber(callback=callback, params=params))
self._subscribers.append(_Subscriber(callback=callback, params=params))

def _add_tag(self, tag: str):
self._tags = _reload("scenario", self)._tags
Expand All @@ -216,7 +216,7 @@ def has_tag(self, tag: str) -> bool:

def _remove_subscriber(self, callback: Callable, params: Optional[List[Any]] = None):
if params is not None:
self._subscribers.remove(Subscriber(callback, params))
self._subscribers.remove(_Subscriber(callback, params))
else:
elem = [x for x in self._subscribers if x.callback == callback]
if not elem:
Expand Down
2 changes: 1 addition & 1 deletion src/taipy/core/task/_task_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ def _bulk_get_or_create(
else:
inputs = [data_nodes[input_config] for input_config in task_config.input_configs]
outputs = [data_nodes[output_config] for output_config in task_config.output_configs]
task = Task(task_config.id, task_config.function, inputs, outputs, parent_id=parent_id)
task = Task(str(task_config.id), task_config.function, inputs, outputs, parent_id=parent_id)
cls._set(task)
tasks.append(task)

Expand Down
4 changes: 2 additions & 2 deletions tests/core/pipeline/test_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

import pytest

from src.taipy.core.common._utils import Subscriber
from src.taipy.core.common._utils import _Subscriber
from src.taipy.core.common.alias import PipelineId, TaskId
from src.taipy.core.data.data_node import DataNode
from src.taipy.core.data.in_memory import InMemoryDataNode
Expand Down Expand Up @@ -271,7 +271,7 @@ def test_auto_set_and_reload(task):
assert len(pipeline_1.subscribers) == 2
assert len(pipeline_2.subscribers) == 2

pipeline_1.subscribers.remove(Subscriber(print, []))
pipeline_1.subscribers.remove(_Subscriber(print, []))
assert len(pipeline_1.subscribers) == 1
assert len(pipeline_2.subscribers) == 1

Expand Down
6 changes: 3 additions & 3 deletions tests/core/pipeline/test_pipeline_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from src.taipy.core._scheduler._scheduler import _Scheduler
from src.taipy.core._scheduler._scheduler_factory import _SchedulerFactory
from src.taipy.core.common import _utils
from src.taipy.core.common._utils import Subscriber
from src.taipy.core.common._utils import _Subscriber
from src.taipy.core.common.alias import PipelineId, TaskId
from src.taipy.core.config.job_config import JobConfig
from src.taipy.core.data._data_manager import _DataManager
Expand Down Expand Up @@ -492,11 +492,11 @@ def test_pipeline_notification_unsubscribe_multi_param():

pipeline.unsubscribe(notify_multi_param)
assert len(pipeline.subscribers) == 2
assert Subscriber(notify_multi_param, ["foobar", 123, 0]) not in pipeline.subscribers
assert _Subscriber(notify_multi_param, ["foobar", 123, 0]) not in pipeline.subscribers

pipeline.unsubscribe(notify_multi_param, ["foobar", 123, 2])
assert len(pipeline.subscribers) == 1
assert Subscriber(notify_multi_param, ["foobar", 123, 2]) not in pipeline.subscribers
assert _Subscriber(notify_multi_param, ["foobar", 123, 2]) not in pipeline.subscribers

with pytest.raises(ValueError):
pipeline.unsubscribe(notify_multi_param, ["foobar", 123, 10000])
Expand Down
Loading

0 comments on commit 36f5d60

Please sign in to comment.