feat : add modules

README.md

@@ -4,30 +4,56 @@ PyVrml是基于python3的开发工具库
 
 提供对各项通用工具的支持，例如Excel，文件操作，网络调用等
 
-用于实现各种工作中需要用到的自动化脚本
+在数据查询方面，您可以通过API或者爬虫的方式，接入您所使用的各种平台
+
+在数据输出方面，支持Excel、Image及各种自定义方式
+
+基于PyVrml定制您的常用工具集，从而便捷的进行数据查询聚合和监控
+
+用于实现各种工作中需要用到的自动化脚本，为自动化监控、大批量数据查询、问题排查等应用场景提供强有力的工具
 
 ## Modules
 
-### excel
+#### ai
+
+封装了机器学习算法，方便快速上手使用。当然，仅供娱乐
+
+#### excel
 
 快速方便的生成excel文档，方便的进行数据统计和文档整理
 
-### file
+#### file
 
 简易的csv文件读取api，便于使用csv进行各项数据脚本的书写
 
-### http
+#### gitlab
+
+通过gitlab的api，获取gitlab上的各项数据
+
+#### http
 
 带有重试功能的http访问api，针对json的包装
 
-### login
+#### login
 
 用于通过登录进行验权，从而爬取受账号权限保护的页面数据
 
-### retry
+#### retry
 
 调用重试功能，避免脚本进行远程调用时失败退出
 
-### time
+#### time
+
+常用时间工具，包括(实践对象)/(字符串)/(时间戳)之间的转换
+
+## 使用示例
+
+`pyvrml`目录下为工具包源码
+
+`demo`目录下为一些自动化脚本demo
+
+您可以参考和`pyvrml.py`文件中的api示例，也可以直接查看`demo`目录下的自动化脚本实例。
+
+## 使用方式
 
-常用时间工具，包括(实践对象)/(字符串)/(时间戳)之间的转换
+请直接下载pyvrml的源码包，将`pyvrml`目录文件放在您自己的开发项目中使用。

pyvrml_demo/csv_to_excel.py → demo/csv_to_excel/csv_to_excel.py

@@ -1,5 +1,5 @@
-from pyvrml.excel.excelhelper import render_excel
-from pyvrml.file.csvhelper import read_csv
+from pyvrml.excel.excelutils import render_excel
+from pyvrml.file.csvutils import read_csv
 
 
 def csv_to_excel_demo1():
@@ -45,7 +45,7 @@ def csv_to_excel_demo2():
                  content_rows=content_rows,
                  excel_name="test2.xls")
 
-
+# 将csv文件转换成execl文件
 if __name__ == '__main__':
     csv_to_excel_demo1()
     csv_to_excel_demo2()

demo/gitlab_monitor/gitlab_monitor.py

@@ -0,0 +1,57 @@
+from pyvrml.gitlab.gitlab import gitlab_auth
+from pyvrml.http.httputils import get
+
+git_token = "token"
+
+# 监控gitlab项目pipeline执行情况
+if __name__ == '__main__':
+    # 登录gitlab
+    gl = gitlab_auth()
+
+    projects_name = ['test']
+    for project_name in projects_name:
+
+        projects = gl.projects.list(search=project_name)
+        for project in projects:
+            print("[project]: " + str(project))
+            search_project_name = project.name
+            if search_project_name != project_name:
+                # 过滤非精确匹配的项目
+                continue
+
+            pipelines = project.pipelines.list(page=1, per_page=1)
+            for pipeline in pipelines:
+                # print("[pipeline1]: " + str(pipeline))
+
+                pipeline = project.pipelines.get(pipeline.id)
+                print("[pipeline]: " + str(pipeline))
+
+                user_name = pipeline.user.get('name')
+                web_url = pipeline.web_url
+                status = pipeline.status
+                # pipeline未通过
+                if status != "success":
+                    print(f"[{project_name}] pipeline健康检查失败!!!")
+
+                # pipeline通过
+                else:
+                    commit_id = pipeline.sha
+                    test_res = get(
+                        url=f"http://git.com/group/{project_name}/commit/{commit_id}/pipeline_reports.json?type=test",
+                        params=None,
+                        headers={'PRIVATE-TOKEN': git_token})
+                    print("[test]: " + str(test_res))
+
+                    if test_res:
+                        summary = test_res.get("summary")
+                        if summary:
+                            total = summary.get("total")
+                            resolved = summary.get("resolved")
+                            failed = summary.get("failed")
+                            error = summary.get("error")
+                            if failed > 0 or error > 0:
+                                print(f"[{project_name}] test健康检查未通过!!!")
+                            else:
+                                print(f"[{project_name}] test健康检查通过~~~")
+                        else:
+                            print(f"[{project_name}] test健康检查获取失败!!!")

pyvrml/ai/prophet/prophet.py

@@ -0,0 +1,220 @@
+import matplotlib
+import numpy as np
+import tensorflow as tf
+from tensorflow.contrib.timeseries.python.timeseries import NumpyReader
+from tensorflow.contrib.timeseries.python.timeseries import estimators as ts_estimators
+from tensorflow.contrib.timeseries.python.timeseries import model as ts_model
+
+matplotlib.use("agg")
+
+import matplotlib.pyplot as plt
+
+
+class _LSTMModel(ts_model.SequentialTimeSeriesModel):
+    """A time series model-building example using an RNNCell."""
+
+    def __init__(self, num_units, num_features, dtype=tf.float32):
+        """Initialize/configure the model object.
+        Note that we do not start graph building here. Rather, this object is a
+        configurable factory for TensorFlow graphs which are run by an Estimator.
+        Args:
+          num_units: The number of units in the model's LSTMCell.
+          num_features: The dimensionality of the time series (features per
+            timestep).
+          dtype: The floating point data type to use.
+        """
+        super(_LSTMModel, self).__init__(
+            # Pre-register the metrics we'll be outputting (just a mean here).
+            train_output_names=["mean"],
+            predict_output_names=["mean"],
+            num_features=num_features,
+            dtype=dtype)
+        self._num_units = num_units
+        # Filled in by initialize_graph()
+        self._lstm_cell = None
+        self._lstm_cell_run = None
+        self._predict_from_lstm_output = None
+
+    def initialize_graph(self, input_statistics):
+        """Save templates for components, which can then be used repeatedly.
+        This method is called every time a new graph is created. It's safe to start
+        adding ops to the current default graph here, but the graph should be
+        constructed from scratch.
+        Args:
+          input_statistics: A math_utils.InputStatistics object.
+        """
+        super(_LSTMModel, self).initialize_graph(input_statistics=input_statistics)
+        self._lstm_cell = tf.nn.rnn_cell.LSTMCell(num_units=self._num_units)
+        # Create templates so we don't have to worry about variable reuse.
+        self._lstm_cell_run = tf.make_template(
+            name_="lstm_cell",
+            func_=self._lstm_cell,
+            create_scope_now_=True)
+        # Transforms LSTM output into mean predictions.
+        self._predict_from_lstm_output = tf.make_template(
+            name_="predict_from_lstm_output",
+            func_=lambda inputs: tf.layers.dense(inputs=inputs, units=self.num_features),
+            create_scope_now_=True)
+
+    def get_start_state(self):
+        """Return initial state for the time series model."""
+        return (
+            # Keeps track of the time associated with this state for error checking.
+            tf.zeros([], dtype=tf.int64),
+            # The previous observation or prediction.
+            tf.zeros([self.num_features], dtype=self.dtype),
+            # The state of the RNNCell (batch dimension removed since this parent
+            # class will broadcast).
+            [tf.squeeze(state_element, axis=0)
+             for state_element
+             in self._lstm_cell.zero_state(batch_size=1, dtype=self.dtype)])
+
+    def _transform(self, data):
+        """Normalize data based on input statistics to encourage stable training."""
+        mean, variance = self._input_statistics.overall_feature_moments
+        return (data - mean) / variance
+
+    def _de_transform(self, data):
+        """Transform data back to the input scale."""
+        mean, variance = self._input_statistics.overall_feature_moments
+        return data * variance + mean
+
+    def _filtering_step(self, current_times, current_values, state, predictions):
+        """Update model state based on observations.
+        Note that we don't do much here aside from computing a loss. In this case
+        it's easier to update the RNN state in _prediction_step, since that covers
+        running the RNN both on observations (from this method) and our own
+        predictions. This distinction can be important for probabilistic models,
+        where repeatedly predicting without filtering should lead to low-confidence
+        predictions.
+        Args:
+          current_times: A [batch size] integer Tensor.
+          current_values: A [batch size, self.num_features] floating point Tensor
+            with new observations.
+          state: The model's state tuple.
+          predictions: The output of the previous `_prediction_step`.
+        Returns:
+          A tuple of new state and a predictions dictionary updated to include a
+          loss (note that we could also return other measures of goodness of fit,
+          although only "loss" will be optimized).
+        """
+        state_from_time, prediction, lstm_state = state
+        with tf.control_dependencies(
+                [tf.assert_equal(current_times, state_from_time)]):
+            transformed_values = self._transform(current_values)
+            # Use mean squared error across features for the loss.
+            predictions["loss"] = tf.reduce_mean(
+                (prediction - transformed_values) ** 2, axis=-1)
+            # Keep track of the new observation in model state. It won't be run
+            # through the LSTM until the next _imputation_step.
+            new_state_tuple = (current_times, transformed_values, lstm_state)
+        return (new_state_tuple, predictions)
+
+    def _prediction_step(self, current_times, state):
+        """Advance the RNN state using a previous observation or prediction."""
+        _, previous_observation_or_prediction, lstm_state = state
+        lstm_output, new_lstm_state = self._lstm_cell_run(
+            inputs=previous_observation_or_prediction, state=lstm_state)
+        next_prediction = self._predict_from_lstm_output(lstm_output)
+        new_state_tuple = (current_times, next_prediction, new_lstm_state)
+        return new_state_tuple, {"mean": self._de_transform(next_prediction)}
+
+    def _imputation_step(self, current_times, state):
+        """Advance model state across a gap."""
+        # Does not do anything special if we're jumping across a gap. More advanced
+        # models, especially probabilistic ones, would want a special case that
+        # depends on the gap size.
+        return state
+
+    def _exogenous_input_step(
+            self, current_times, current_exogenous_regressors, state):
+        """Update model state based on exogenous regressors."""
+        raise NotImplementedError(
+            "Exogenous inputs are not implemented for this example.")
+
+
+def prophet_lstm_values(values_y: [], save_img=False, pic_name='predict_result.png', window_size=100):
+    """
+    使用lstm预测一维值序列
+    :param values_y: 值序列
+    :param save_img: 是否保存图片？否的话返回plt对象
+    :param pic_name: 保存名称
+    :param window_size: 最小计算窗口值
+    :return: 若未保存图片，返回plt对象
+    """
+    times_x = [i for i in range(len(values_y))]
+    return prophet_lstm(times_x, values_y, save_img=save_img, pic_name=pic_name, window_size=window_size)
+
+
+def prophet_lstm(times_x: [], values_y: [], save_img=False, pic_name='predict_result.png', window_size=100):
+    """
+    使用lstm预测二维时间值序列
+    :param times_x: 时间序列
+    :param values_y: 值序列
+    :param save_img: 是否保存图片？否的话返回plt对象
+    :param pic_name: 保存名称
+    :param window_size: 最小计算窗口值
+    :return: 若未保存图片，返回plt对象
+    """
+    times_x = np.array(times_x)
+    values_y = np.array(values_y)
+    tf.logging.set_verbosity(tf.logging.INFO)
+
+    data = {
+        tf.contrib.timeseries.TrainEvalFeatures.TIMES: times_x,
+        tf.contrib.timeseries.TrainEvalFeatures.VALUES: values_y,
+    }
+
+    reader = NumpyReader(data)
+
+    train_input_fn = tf.contrib.timeseries.RandomWindowInputFn(reader,
+                                                               batch_size=4,
+                                                               window_size=window_size)
+
+    estimator = ts_estimators.TimeSeriesRegressor(model=_LSTMModel(num_features=1,
+                                                                   num_units=128),
+                                                  optimizer=tf.train.AdamOptimizer(0.001))
+    estimator.train(input_fn=train_input_fn,
+                    steps=2000)
+
+    evaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)
+    evaluation = estimator.evaluate(input_fn=evaluation_input_fn,
+                                    steps=1)
+
+    # Predict starting after the evaluation
+    (predictions,) = tuple(estimator.predict(input_fn=
+                                             tf.contrib.timeseries.predict_continuation_input_fn(evaluation,
+                                                                                                 steps=200)))
+
+    observed_times = evaluation["times"][0]
+    observed = evaluation["observed"][0, :, :]
+    evaluated_times = evaluation["times"][0]
+    evaluated = evaluation["mean"][0]
+    predicted_times = predictions['times']
+    predicted = predictions["mean"]
+
+    plt.figure(figsize=(15, 5))
+    plt.axvline(999,
+                linestyle="dotted",
+                linewidth=4,
+                color='r')
+    observed_lines = plt.plot(observed_times,
+                              observed,
+                              label="observation",
+                              color="k")
+    evaluated_lines = plt.plot(evaluated_times,
+                               evaluated,
+                               label="evaluation",
+                               color="g")
+    predicted_lines = plt.plot(predicted_times,
+                               predicted,
+                               label="prediction",
+                               color="r")
+    plt.legend(handles=[observed_lines[0],
+                        evaluated_lines[0],
+                        predicted_lines[0]],
+               loc="upper left")
+
+    if save_img:
+        plt.savefig(pic_name)
+    return plt