diff --git a/demo/components/roc_curve_test.py b/demo/components/roc_curve_test.py new file mode 100644 index 000000000..42440dba8 --- /dev/null +++ b/demo/components/roc_curve_test.py @@ -0,0 +1,27 @@ +# Copyright (c) 2020 VisualDL Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ======================================================================= +# coding=utf-8 +from visualdl import LogWriter +import numpy as np + +with LogWriter("./log/roc_curve_test/train") as writer: + for step in range(3): + labels = np.random.randint(2, size=100) + predictions = np.random.rand(100) + writer.add_roc_curve(tag='roc_curve', + labels=labels, + predictions=predictions, + step=step, + num_thresholds=5) diff --git a/frontend/packages/core/public/locales/en/common.json b/frontend/packages/core/public/locales/en/common.json index ed0dcc737..097423478 100644 --- a/frontend/packages/core/public/locales/en/common.json +++ b/frontend/packages/core/public/locales/en/common.json @@ -13,6 +13,7 @@ "loading": "Please wait while loading data", "next-page": "Next Page", "pr-curve": "PR-Curve", + "roc-curve": "ROC-Curve", "previous-page": "Prev Page", "run": "Run", "running": "Running", diff --git a/frontend/packages/core/public/locales/zh/common.json b/frontend/packages/core/public/locales/zh/common.json index 0d96a19c1..6711dd136 100644 --- a/frontend/packages/core/public/locales/zh/common.json +++ b/frontend/packages/core/public/locales/zh/common.json @@ -13,6 +13,7 @@ "loading": "数据载入中，请稍等", "next-page": "下一页", "pr-curve": "PR曲线", + "roc-curve": "ROC曲线", "previous-page": "上一页", "run": "运行", "running": "运行中", diff --git a/frontend/packages/core/src/components/ROC_CurvePage/ROC_CurveChart.tsx b/frontend/packages/core/src/components/ROC_CurvePage/ROC_CurveChart.tsx new file mode 100644 index 000000000..4818b8edd --- /dev/null +++ b/frontend/packages/core/src/components/ROC_CurvePage/ROC_CurveChart.tsx @@ -0,0 +1,232 @@ +/** + * Copyright 2020 Baidu Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import LineChart, {LineChartRef} from '~/components/LineChart'; +import type {ROC_CurveData, Run} from '~/resource/roc-curve'; +import React, {FunctionComponent, useCallback, useMemo, useRef, useState} from 'react'; +import {options as chartOptions, nearestPoint} from '~/resource/roc-curve'; +import {rem, size} from '~/utils/style'; + +import ChartToolbox from '~/components/ChartToolbox'; +import type {EChartOption} from 'echarts'; +import TooltipTable from '~/components/TooltipTable'; +import {cycleFetcher} from '~/utils/fetch'; +import ee from '~/utils/event'; +import {format} from 'd3-format'; +import queryString from 'query-string'; +import {renderToStaticMarkup} from 'react-dom/server'; +import styled from 'styled-components'; +import {useRunningRequest} from '~/hooks/useRequest'; +import {useTranslation} from 'react-i18next'; +import zip from 'lodash/zip'; + +const axisFormatter = format('.4f'); +const valueFormatter = format('.2f'); + +const Wrapper = styled.div` + ${size('100%', '100%')} + display: flex; + flex-direction: column; + align-items: stretch; + justify-content: space-between; +`; + +const StyledLineChart = styled(LineChart)` + flex-grow: 1; +`; + +const Toolbox = styled(ChartToolbox)` + margin-left: ${rem(20)}; + margin-right: ${rem(20)}; + margin-bottom: ${rem(18)}; +`; + +const Error = styled.div` + ${size('100%', '100%')} + display: flex; + justify-content: center; + align-items: center; +`; + +type ROC_CurveChartProps = { + cid: symbol; + runs: Run[]; + tag: string; + running?: boolean; +}; + +const ROC_CurveChart: FunctionComponent = ({cid, runs, tag, running}) => { + const {t} = useTranslation(['roc-curve', 'common']); + + const echart = useRef(null); + + const {data: dataset, error, loading} = useRunningRequest( + runs.map(run => `/roc-curve/list?${queryString.stringify({run: run.label, tag})}`), + !!running, + (...urls) => cycleFetcher(urls) + ); + + const [maximized, setMaximized] = useState(false); + const toggleMaximized = useCallback(() => { + ee.emit('toggle-chart-size', cid, !maximized); + setMaximized(m => !m); + }, [cid, maximized]); + + const selectedData = useMemo<[number, number, number[][]][]>( + () => + runs.map((run, i) => { + const [wallTime, step, ...item] = dataset?.[i]?.find(row => row[1] === run.steps[run.index]) ?? [ + 0, + 0, + [], + [], + [], + [], + [], + [], + [] + ]; + return [wallTime, step, zip(...item) as number[][]]; + }), + [dataset, runs] + ); + + const data = useMemo( + () => + selectedData.map((item, i) => { + const run = runs[i]; + return { + name: run.label, + z: i, + itemStyle: { + color: run.colors[0] + }, + lineStyle: { + color: run.colors[0] + }, + data: item[2], + encode: { + x: [1], + y: [0] + } + }; + }), + [selectedData, runs] + ); + + const formatter = useCallback( + (params: EChartOption.Tooltip.Format | EChartOption.Tooltip.Format[]) => { + const series = Array.isArray(params) ? params[0].data : params.data; + const points = nearestPoint( + selectedData.map(s => s[2]), + series[1] + ); + const columns = [ + { + label: t('roc-curve:threshold') + }, + { + label: t('roc-curve:tpr') + }, + { + label: t('roc-curve:fpr') + }, + { + label: t('roc-curve:true-positives') + }, + { + label: t('roc-curve:false-positives') + }, + { + label: t('roc-curve:true-negatives') + }, + { + label: t('roc-curve:false-negatives') + } + ]; + const runData = points.reduce((m, runPoints, index) => { + m.push(...new Array(runPoints.length).fill(runs[index])); + return m; + }, []); + const data = points.reduce<(string | number)[][]>((m, runPoints) => { + m.push( + ...runPoints.map(([tpr, fpr, tp, fp, tn, fn, threshold]) => [ + valueFormatter(threshold), + axisFormatter(tpr), + axisFormatter(fpr), + tp, + fp, + tn, + fn + ]) + ); + return m; + }, []); + return renderToStaticMarkup( + + ); + }, + [selectedData, runs, t] + ); + + const options = useMemo( + () => ({ + ...chartOptions, + tooltip: { + ...chartOptions.tooltip, + formatter, + hideDelay: 300, + enterable: true + } + }), + [formatter] + ); + + // display error only on first fetch + if (!data && error) { + return {t('common:error')}; + } + + return ( + + + echart.current?.restore() + }, + { + icon: 'download', + tooltip: t('roc-curve:download-image'), + onClick: () => echart.current?.saveAsImage() + } + ]} + /> + + ); +}; + +export default ROC_CurveChart; diff --git a/frontend/packages/core/src/components/ROC_CurvePage/StepSlider.tsx b/frontend/packages/core/src/components/ROC_CurvePage/StepSlider.tsx new file mode 100644 index 000000000..2c657ce9f --- /dev/null +++ b/frontend/packages/core/src/components/ROC_CurvePage/StepSlider.tsx @@ -0,0 +1,110 @@ +/** + * Copyright 2020 Baidu Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React, {FunctionComponent, useEffect, useState} from 'react'; +import {ellipsis, size, transitionProps} from '~/utils/style'; + +import Field from '~/components/Field'; +import RangeSlider from '~/components/RangeSlider'; +import type {Run} from '~/resource/roc-curve'; +import {TimeType} from '~/resource/roc-curve'; +import {format} from 'd3-format'; +import {formatTime} from '~/utils'; +import styled from 'styled-components'; +import {useTranslation} from 'react-i18next'; + +const relativeFormatter = format('.2f'); + +const TimeDisplay = styled.div` + color: var(--text-lighter-color); + font-size: 0.857142857em; + padding-left: 1.666666667em; + margin-bottom: 0.416666667em; + ${transitionProps('color')} +`; + +const Label = styled.span<{color: string}>` + display: inline-block; + padding-left: 1.428571429em; + position: relative; + ${ellipsis()} + + &::before { + content: ''; + display: block; + ${size('0.857142857em', '0.857142857em')} + background-color: ${props => props.color}; + border-radius: 50%; + position: absolute; + left: 0; + top: 50%; + transform: translateY(-50%); + } +`; + +const FullWidthRangeSlider = styled(RangeSlider)` + width: 100%; +`; + +const typeMap = { + [TimeType.WallTime]: 'wallTimes', + [TimeType.Relative]: 'relatives', + [TimeType.Step]: 'steps' +} as const; + +const formatter = { + [TimeType.WallTime]: (wallTime: number, {i18n}: ReturnType) => + formatTime(wallTime, i18n.language), + [TimeType.Relative]: (relative: number) => `${relativeFormatter(relative)} ms`, + [TimeType.Step]: (step: number, {t}: ReturnType) => `${t('common:time-mode.step')} ${step}` +} as const; + +type StepSliderProps = { + run: Run; + type: TimeType; + onChange?: (step: number) => unknown; +}; + +const StepSlider: FunctionComponent = ({onChange, run, type}) => { + const translation = useTranslation('common'); + + const [index, setIndex] = useState(run.index); + useEffect(() => setIndex(run.index), [run.index]); + + return ( + + {run.label} + + } + > + + {run[typeMap[type]][index] == null ? '...' : formatter[type](run[typeMap[type]][index], translation)} + + onChange?.(index)} + /> + + ); +}; + +export default StepSlider; diff --git a/frontend/packages/core/src/hooks/useNavItems.ts b/frontend/packages/core/src/hooks/useNavItems.ts index 16a650956..ce6fdadcc 100644 --- a/frontend/packages/core/src/hooks/useNavItems.ts +++ b/frontend/packages/core/src/hooks/useNavItems.ts @@ -28,7 +28,8 @@ export const navMap = { audio: Pages.Audio, graph: Pages.Graph, embeddings: Pages.HighDimensional, - pr_curve: Pages.PRCurve + pr_curve: Pages.PRCurve, + roc_curve: Pages.ROC_Curve } as const; const useNavItems = () => { diff --git a/frontend/packages/core/src/pages/roc-curve.tsx b/frontend/packages/core/src/pages/roc-curve.tsx new file mode 100644 index 000000000..320577334 --- /dev/null +++ b/frontend/packages/core/src/pages/roc-curve.tsx @@ -0,0 +1,177 @@ +/** + * Copyright 2020 Baidu Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ChartPage, {WithChart} from '~/components/ChartPage'; +import React, {FunctionComponent, useCallback, useEffect, useMemo, useState} from 'react'; +import type {Run, StepInfo, Tag} from '~/resource/roc-curve'; +import {rem, transitionProps} from '~/utils/style'; + +import {AsideSection} from '~/components/Aside'; +import Content from '~/components/Content'; +import Error from '~/components/Error'; +import Field from '~/components/Field'; +import ROC_CurveChart from '~/components/ROC_CurvePage/ROC_CurveChart'; +import RunAside from '~/components/RunAside'; +import StepSlider from '~/components/ROC_CurvePage/StepSlider'; +import TimeModeSelect from '~/components/TimeModeSelect'; +import {TimeType} from '~/resource/roc-curve'; +import Title from '~/components/Title'; +import {cycleFetcher} from '~/utils/fetch'; +import queryString from 'query-string'; +import styled from 'styled-components'; +import {useRunningRequest} from '~/hooks/useRequest'; +import useTagFilter from '~/hooks/useTagFilter'; +import {useTranslation} from 'react-i18next'; + +const StepSliderWrapper = styled.div` + max-height: 30vh; + overflow: auto; + overflow-x: hidden; + overflow-y: auto; + flex-shrink: 0; + + > ${AsideSection}:last-child { + padding-bottom: ${rem(20)}; + margin-bottom: 0; + } + + + .run-section { + border-top: 1px solid var(--border-color); + margin-top: 0; + padding-top: ${rem(20)}; + ${transitionProps('border-color')} + } + + &:empty + .run-section { + border-top: none; + } +`; + +const ROC_Curve: FunctionComponent = () => { + const {t} = useTranslation(['roc-curve', 'common']); + + const [running, setRunning] = useState(true); + + const {runs, tags, runsInTags, selectedRuns, onChangeRuns, loading} = useTagFilter('roc-curve', running); + + const [indexes, setIndexes] = useState>({}); + const onChangeIndexes = useCallback( + (run: string, index: number) => + setIndexes(indexes => ({ + ...indexes, + [run]: index + })), + [] + ); + useEffect( + () => + setIndexes(indexes => + runsInTags.reduce((m, c) => { + if (indexes[c.label] != null) { + m[c.label] = indexes[c.label]; + } + return m; + }, {}) + ), + [runsInTags] + ); + + const {data: stepInfo} = useRunningRequest( + runsInTags.map(run => `/roc-curve/steps?${queryString.stringify({run: run.label})}`), + !!running, + (...urls) => cycleFetcher(urls) + ); + const runWithInfo = useMemo( + () => + runsInTags.map((run, i) => ({ + ...run, + index: indexes[run.label] ?? (stepInfo?.[i].length ?? 1) - 1, + steps: stepInfo?.[i].map(j => j[1]) ?? [], + wallTimes: stepInfo?.[i].map(j => Math.floor(j[0])) ?? [], + relatives: stepInfo?.[i].map(j => j[0] - stepInfo[i][0][0]) ?? [] + })), + [runsInTags, stepInfo, indexes] + ); + + const [timeType, setTimeType] = useState(TimeType.Step); + + const rocCurveTags = useMemo( + () => + tags.map(tag => ({ + ...tag, + runs: tag.runs.map(run => ({ + ...run, + index: 0, + steps: [] as Run['steps'], + wallTimes: [] as Run['wallTimes'], + relatives: [] as Run['relatives'], + ...runWithInfo.find(r => r.label === run.label) + })) + })), + [tags, runWithInfo] + ); + + const aside = useMemo( + () => + runs.length ? ( + + + + + + + + {runWithInfo.map(run => ( + + onChangeIndexes(run.label, index)} + /> + + ))} + + + ) : null, + [t, onChangeRuns, running, runs, selectedRuns, timeType, runWithInfo, onChangeIndexes] + ); + + const withChart = useCallback>( + ({label, runs, ...args}) => , + [running] + ); + + return ( + <> + {t('common:roc-curve')} + + {!loading && !runs.length ? ( + + ) : ( + + )} + + + ); +}; + +export default ROC_Curve; diff --git a/frontend/packages/core/src/resource/roc-curve/chart.ts b/frontend/packages/core/src/resource/roc-curve/chart.ts new file mode 100644 index 000000000..17935d16e --- /dev/null +++ b/frontend/packages/core/src/resource/roc-curve/chart.ts @@ -0,0 +1,27 @@ +/** + * Copyright 2020 Baidu Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const options = { + legend: { + data: [] + }, + tooltip: { + position: ['10%', '100%'] + }, + yAxis: { + splitNumber: 5 + } +}; diff --git a/frontend/packages/core/src/resource/roc-curve/data.ts b/frontend/packages/core/src/resource/roc-curve/data.ts new file mode 100644 index 000000000..2bc37f926 --- /dev/null +++ b/frontend/packages/core/src/resource/roc-curve/data.ts @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Baidu Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const nearestPoint = (data: number[][][], recall: number): number[][][] => { + return data.map(series => { + let delta = Number.POSITIVE_INFINITY; + let nearestRecall = 0; + for (let i = 0; i < series.length; i++) { + const d = Math.abs(series[i][1] - recall); + if (d < Number.EPSILON) { + nearestRecall = series[i][1]; + break; + } + if (d < delta) { + delta = d; + nearestRecall = series[i][1]; + } + } + return series.filter(s => s[1] === nearestRecall); + }); +}; diff --git a/frontend/packages/core/src/resource/roc-curve/index.ts b/frontend/packages/core/src/resource/roc-curve/index.ts new file mode 100644 index 000000000..4b2ad5c83 --- /dev/null +++ b/frontend/packages/core/src/resource/roc-curve/index.ts @@ -0,0 +1,20 @@ +/** + * Copyright 2020 Baidu Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type {ROC_CurveData, Run, StepInfo, Tag} from './types'; +export {TimeType} from './types'; +export * from './chart'; +export * from './data'; diff --git a/frontend/packages/core/src/resource/roc-curve/types.ts b/frontend/packages/core/src/resource/roc-curve/types.ts new file mode 100644 index 000000000..088f56105 --- /dev/null +++ b/frontend/packages/core/src/resource/roc-curve/types.ts @@ -0,0 +1,54 @@ +/** + * Copyright 2020 Baidu Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {Run as BaseRun, Tag as BaseTag, TimeMode} from '~/types'; + +export {TimeMode as TimeType}; + +type Step = number; +type WallTime = number; +type Relative = number; +type Tpr = number; +type Fpr = number; +type TruePositives = number; +type FalsePositives = number; +type TrueNegatives = number; +type FalseNegatives = number; +type Thresholds = number; + +export type PRCurveDataItem = [ + WallTime, + Step, + Tpr[], + Fpr[], + TruePositives[], + FalsePositives[], + TrueNegatives[], + FalseNegatives[], + Thresholds[] +]; +export type ROC_CurveData = ROC_CurveDataItem[]; + +export interface Run extends BaseRun { + index: number; + steps: Step[]; + wallTimes: WallTime[]; + relatives: Relative[]; +} + +export type Tag = BaseTag; + +export type StepInfo = [WallTime, Step][]; diff --git a/frontend/packages/core/src/routes/index.ts b/frontend/packages/core/src/routes/index.ts index a4b5c3f2b..89855be26 100644 --- a/frontend/packages/core/src/routes/index.ts +++ b/frontend/packages/core/src/routes/index.ts @@ -23,7 +23,8 @@ export enum Pages { Audio = 'audio', Graph = 'graph', HighDimensional = 'high-dimensional', - PRCurve = 'pr-curve' + PRCurve = 'pr-curve', + ROC_Curve = 'roc-curve' } export interface Route { @@ -82,6 +83,11 @@ const routes: Route[] = [ id: Pages.PRCurve, path: '/pr-curve', component: React.lazy(() => import('~/pages/pr-curve')) + }, + { + id: Pages.ROC_Curve, + path: '/roc-curve', + component: React.lazy(() => import('~/pages/roc-curve')) } ]; diff --git a/frontend/packages/core/src/store/runs/reducers.ts b/frontend/packages/core/src/store/runs/reducers.ts index a8ac17898..bf76b6c51 100644 --- a/frontend/packages/core/src/store/runs/reducers.ts +++ b/frontend/packages/core/src/store/runs/reducers.ts @@ -23,7 +23,8 @@ const initState: RunsState = { histogram: [], image: [], audio: [], - 'pr-curve': [] + 'pr-curve': [], + 'roc-curve': [] }; function runsReducer(state = initState, action: RunsActionTypes): RunsState { diff --git a/frontend/packages/core/src/store/runs/types.ts b/frontend/packages/core/src/store/runs/types.ts index bff16a95a..5e5661151 100644 --- a/frontend/packages/core/src/store/runs/types.ts +++ b/frontend/packages/core/src/store/runs/types.ts @@ -26,6 +26,7 @@ export interface RunsState { image: Runs; audio: Runs; 'pr-curve': Runs; + 'roc-curve': Runs; } export type Page = keyof RunsState; diff --git a/frontend/packages/demo/builder/index.ts b/frontend/packages/demo/builder/index.ts index 42731ba5d..dc154c5de 100644 --- a/frontend/packages/demo/builder/index.ts +++ b/frontend/packages/demo/builder/index.ts @@ -27,7 +27,7 @@ import {spawn} from 'child_process'; const host = '127.0.0.1'; const publicPath = '/visualdl'; -const pages = ['common', 'scalar', 'histogram', 'image', 'audio', 'graph', 'pr-curve', 'high-dimensional']; +const pages = ['common', 'scalar', 'histogram', 'image', 'audio', 'graph', 'pr-curve', 'roc-curve', 'high-dimensional']; const dataDir = path.resolve(__dirname, '../data'); async function start() { diff --git a/frontend/packages/demo/builder/roc-curve.ts b/frontend/packages/demo/builder/roc-curve.ts new file mode 100644 index 000000000..b63060587 --- /dev/null +++ b/frontend/packages/demo/builder/roc-curve.ts @@ -0,0 +1,34 @@ +/** + * Copyright 2020 Baidu Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type {Data, Worker} from './types'; + +const worker: Worker = async io => { + const components = await io.getData('/components'); + if (!components.includes('roc_curve')) { + return; + } + + const {runs, tags} = await io.save('/roc-curve/tags'); + for (const [index, run] of runs.entries()) { + await io.save('/roc-curve/steps', {run}); + for (const tag of tags[index]) { + await io.save('/roc-curve/list', {run, tag}); + } + } +}; + +export default worker; diff --git a/frontend/packages/mock/data/components.ts b/frontend/packages/mock/data/components.ts index 2fcc7c4ed..94ca3d17f 100644 --- a/frontend/packages/mock/data/components.ts +++ b/frontend/packages/mock/data/components.ts @@ -14,4 +14,4 @@ * limitations under the License. */ -export default ['embeddings', 'scalar', 'image', 'audio', 'graph', 'histogram', 'pr_curve']; +export default ['embeddings', 'scalar', 'image', 'audio', 'graph', 'histogram', 'pr_curve', 'roc_curve']; diff --git a/frontend/packages/mock/data/roc-curve/list.ts b/frontend/packages/mock/data/roc-curve/list.ts new file mode 100644 index 000000000..e89550061 --- /dev/null +++ b/frontend/packages/mock/data/roc-curve/list.ts @@ -0,0 +1,6399 @@ +/** + * Copyright 2020 Baidu Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {Request} from 'express'; + +export default (request: Request) => { + if (request.query.run === 'train') { + return [ + [ + 1593069993786.464, + 0, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3348623812198639, + 0.35121950507164, + 0.3604061007499695, + 0.3729729652404785, + 0.3720930218696594, + 0.3899371027946472, + 0.3986486494541168, + 0.40740740299224854, + 0.4065040647983551, + 0.43396225571632385, + 0.47252747416496277, + 0.47058823704719543, + 0.5066666603088379, + 0.48571428656578064, + 0.5, + 0.5319148898124695, + 0.5121951103210449, + 0.5555555820465088, + 0.692307710647583, + 0.739130437374115, + 0.761904776096344, + 0.8125, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0.9866666793823242, + 0.9733333587646484, + 0.9599999785423279, + 0.9466666579246521, + 0.9200000166893005, + 0.8533333539962769, + 0.8266666531562805, + 0.7866666913032532, + 0.7333333492279053, + 0.6666666865348816, + 0.6133333444595337, + 0.5733333230018616, + 0.5333333611488342, + 0.5066666603088379, + 0.4533333480358124, + 0.41333332657814026, + 0.3333333432674408, + 0.2800000011920929, + 0.2666666805744171, + 0.23999999463558197, + 0.2266666740179062, + 0.2133333384990692, + 0.1733333319425583, + 0.1599999964237213, + 0.13333334028720856, + 0.1066666692495346, + 0.09333333373069763, + 0.07999999821186066, + 0.07999999821186066, + 0.07999999821186066, + 0.06666667014360428, + 0.0533333346247673, + 0.02666666731238365, + 0.013333333656191826 + ], + [ + 75, + 75, + 75, + 75, + 75, + 75, + 75, + 74, + 73, + 72, + 71, + 69, + 64, + 62, + 59, + 55, + 50, + 46, + 43, + 40, + 38, + 34, + 31, + 25, + 21, + 20, + 18, + 17, + 16, + 13, + 12, + 10, + 8, + 7, + 6, + 6, + 6, + 5, + 4, + 2, + 1 + ], + [ + 150, + 150, + 150, + 150, + 150, + 150, + 150, + 148, + 145, + 133, + 126, + 116, + 108, + 97, + 89, + 80, + 73, + 60, + 48, + 45, + 37, + 36, + 31, + 22, + 20, + 16, + 8, + 6, + 5, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2, + 5, + 17, + 24, + 34, + 42, + 53, + 61, + 70, + 77, + 90, + 102, + 105, + 113, + 114, + 119, + 128, + 130, + 134, + 142, + 144, + 145, + 147, + 150, + 150, + 150, + 150, + 150, + 150, + 150, + 150, + 150, + 150, + 150 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 2, + 3, + 4, + 6, + 11, + 13, + 16, + 20, + 25, + 29, + 32, + 35, + 37, + 41, + 44, + 50, + 54, + 55, + 57, + 58, + 59, + 62, + 63, + 65, + 67, + 68, + 69, + 69, + 69, + 70, + 71, + 73, + 74 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82 + ] + ], + [ + 1593069993787.353, + 1, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3348214328289032, + 0.3333333432674408, + 0.3509615361690521, + 0.36000001430511475, + 0.3776595890522003, + 0.395480215549469, + 0.386904776096344, + 0.40816327929496765, + 0.42335766553878784, + 0.4193548262119293, + 0.4285714328289032, + 0.43119266629219055, + 0.4536082446575165, + 0.46666666865348816, + 0.4749999940395355, + 0.4864864945411682, + 0.46875, + 0.4912280738353729, + 0.52173912525177, + 0.4871794879436493, + 0.5, + 0.4482758641242981, + 0.4399999976158142, + 0.40909090638160706, + 0.4117647111415863, + 0.4375, + 0.4166666567325592, + 0.5555555820465088, + 0.5, + 0.5, + 0.25, + 0.5, + 0.5, + 0.5, + 0, + 0, + 0 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0.9733333587646484, + 0.9733333587646484, + 0.9599999785423279, + 0.9466666579246521, + 0.9333333373069763, + 0.8666666746139526, + 0.800000011920929, + 0.7733333110809326, + 0.6933333277702332, + 0.6399999856948853, + 0.6266666650772095, + 0.5866666436195374, + 0.5600000023841858, + 0.5066666603088379, + 0.47999998927116394, + 0.4000000059604645, + 0.3733333349227905, + 0.3199999928474426, + 0.25333333015441895, + 0.23999999463558197, + 0.1733333319425583, + 0.14666666090488434, + 0.11999999731779099, + 0.09333333373069763, + 0.09333333373069763, + 0.06666667014360428, + 0.06666667014360428, + 0.0533333346247673, + 0.03999999910593033, + 0.013333333656191826, + 0.013333333656191826, + 0.013333333656191826, + 0.013333333656191826, + 0, + 0, + 0 + ], + [ + 75, + 75, + 75, + 75, + 75, + 75, + 75, + 73, + 73, + 72, + 71, + 70, + 65, + 60, + 58, + 52, + 48, + 47, + 44, + 42, + 38, + 36, + 30, + 28, + 24, + 19, + 18, + 13, + 11, + 9, + 7, + 7, + 5, + 5, + 4, + 3, + 1, + 1, + 1, + 1, + 0, + 0, + 0 + ], + [ + 150, + 150, + 150, + 150, + 150, + 150, + 149, + 146, + 135, + 128, + 117, + 107, + 103, + 87, + 79, + 72, + 64, + 62, + 53, + 48, + 42, + 38, + 34, + 29, + 22, + 20, + 18, + 16, + 14, + 13, + 10, + 9, + 7, + 4, + 4, + 3, + 3, + 1, + 1, + 1, + 1, + 1, + 1 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 4, + 15, + 22, + 33, + 43, + 47, + 63, + 71, + 78, + 86, + 88, + 97, + 102, + 108, + 112, + 116, + 121, + 128, + 130, + 132, + 134, + 136, + 137, + 140, + 141, + 143, + 146, + 146, + 147, + 147, + 149, + 149, + 149, + 149, + 149, + 149 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2, + 2, + 3, + 4, + 5, + 10, + 15, + 17, + 23, + 27, + 28, + 31, + 33, + 37, + 39, + 45, + 47, + 51, + 56, + 57, + 62, + 64, + 66, + 68, + 68, + 70, + 70, + 71, + 72, + 74, + 74, + 74, + 74, + 75, + 75, + 75 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82, + 0.84, + 0.86 + ] + ], + [ + 1593069993788.1448, + 2, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3348214328289032, + 0.33181819319725037, + 0.33796295523643494, + 0.34272301197052, + 0.3497537076473236, + 0.3526315689086914, + 0.36781609058380127, + 0.3836477994918823, + 0.3959731459617615, + 0.3893129825592041, + 0.3613445460796356, + 0.37383177876472473, + 0.3814432919025421, + 0.42352941632270813, + 0.43661972880363464, + 0.4029850661754608, + 0.4166666567325592, + 0.4038461446762085, + 0.41304346919059753, + 0.4749999940395355, + 0.4571428596973419, + 0.4137931168079376, + 0.3913043439388275, + 0.44999998807907104, + 0.47058823704719543, + 0.5, + 0.5384615659713745, + 0.7777777910232544, + 0.8571428656578064, + 0.800000011920929, + 0.75, + 0.75, + 0.6666666865348816, + 0.5, + 0, + 0 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 0.9733333587646484, + 0.9733333587646484, + 0.9733333587646484, + 0.9466666579246521, + 0.8933333158493042, + 0.8533333539962769, + 0.8133333325386047, + 0.7866666913032532, + 0.6800000071525574, + 0.5733333230018616, + 0.5333333611488342, + 0.4933333396911621, + 0.47999998927116394, + 0.41333332657814026, + 0.36000001430511475, + 0.3333333432674408, + 0.2800000011920929, + 0.25333333015441895, + 0.25333333015441895, + 0.2133333384990692, + 0.1599999964237213, + 0.11999999731779099, + 0.11999999731779099, + 0.1066666692495346, + 0.09333333373069763, + 0.09333333373069763, + 0.09333333373069763, + 0.07999999821186066, + 0.0533333346247673, + 0.03999999910593033, + 0.03999999910593033, + 0.02666666731238365, + 0.013333333656191826, + 0, + 0 + ], + [ + 75, + 75, + 75, + 75, + 75, + 75, + 73, + 73, + 73, + 71, + 67, + 64, + 61, + 59, + 51, + 43, + 40, + 37, + 36, + 31, + 27, + 25, + 21, + 19, + 19, + 16, + 12, + 9, + 9, + 8, + 7, + 7, + 7, + 6, + 4, + 3, + 3, + 2, + 1, + 0, + 0 + ], + [ + 150, + 150, + 150, + 150, + 150, + 149, + 147, + 143, + 140, + 132, + 123, + 110, + 98, + 90, + 80, + 76, + 67, + 60, + 49, + 40, + 40, + 35, + 31, + 27, + 21, + 19, + 17, + 14, + 11, + 9, + 7, + 6, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + [ + 0, + 0, + 0, + 0, + 0, + 1, + 3, + 7, + 10, + 18, + 27, + 40, + 52, + 60, + 70, + 74, + 83, + 90, + 101, + 110, + 110, + 115, + 119, + 123, + 129, + 131, + 133, + 136, + 139, + 141, + 143, + 144, + 148, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + 149 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 2, + 2, + 2, + 4, + 8, + 11, + 14, + 16, + 24, + 32, + 35, + 38, + 39, + 44, + 48, + 50, + 54, + 56, + 56, + 59, + 63, + 66, + 66, + 67, + 68, + 68, + 68, + 69, + 71, + 72, + 72, + 73, + 74, + 75, + 75 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82 + ] + ], + [ + 1593069993788.836, + 3, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3303571343421936, + 0.32407405972480774, + 0.32380953431129456, + 0.3350253701210022, + 0.34224599599838257, + 0.36627906560897827, + 0.3459119498729706, + 0.36986300349235535, + 0.375, + 0.3739837408065796, + 0.38532111048698425, + 0.4040403962135315, + 0.42696627974510193, + 0.45121949911117554, + 0.47826087474823, + 0.49152541160583496, + 0.49056604504585266, + 0.4897959232330322, + 0.4651162922382355, + 0.4749999940395355, + 0.5, + 0.48275861144065857, + 0.5, + 0.4761904776096344, + 0.5, + 0.47058823704719543, + 0.5714285969734192, + 0.5454545617103577, + 0.5555555820465088, + 0.5, + 0.5714285969734192, + 0.5714285969734192, + 0.25, + 0.3333333432674408, + 0, + 0, + 0, + 0, + 0 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 0.9866666793823242, + 0.9333333373069763, + 0.9066666960716248, + 0.8799999952316284, + 0.8533333539962769, + 0.8399999737739563, + 0.7333333492279053, + 0.7200000286102295, + 0.6800000071525574, + 0.6133333444595337, + 0.5600000023841858, + 0.5333333611488342, + 0.5066666603088379, + 0.4933333396911621, + 0.4399999976158142, + 0.3866666555404663, + 0.3466666638851166, + 0.3199999928474426, + 0.2666666805744171, + 0.25333333015441895, + 0.23999999463558197, + 0.18666666746139526, + 0.14666666090488434, + 0.13333334028720856, + 0.13333334028720856, + 0.1066666692495346, + 0.1066666692495346, + 0.07999999821186066, + 0.06666667014360428, + 0.0533333346247673, + 0.0533333346247673, + 0.0533333346247673, + 0.013333333656191826, + 0.013333333656191826, + 0, + 0, + 0, + 0, + 0 + ], + [ + 75, + 75, + 75, + 75, + 75, + 75, + 74, + 70, + 68, + 66, + 64, + 63, + 55, + 54, + 51, + 46, + 42, + 40, + 38, + 37, + 33, + 29, + 26, + 24, + 20, + 19, + 18, + 14, + 11, + 10, + 10, + 8, + 8, + 6, + 5, + 4, + 4, + 4, + 1, + 1, + 0, + 0, + 0, + 0, + 0 + ], + [ + 150, + 150, + 150, + 150, + 150, + 150, + 150, + 146, + 142, + 131, + 123, + 109, + 104, + 92, + 85, + 77, + 67, + 59, + 51, + 45, + 36, + 30, + 27, + 25, + 23, + 21, + 18, + 15, + 11, + 11, + 10, + 9, + 6, + 5, + 4, + 4, + 3, + 3, + 3, + 2, + 2, + 1, + 1, + 1, + 1 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4, + 8, + 19, + 27, + 41, + 46, + 58, + 65, + 73, + 83, + 91, + 99, + 105, + 114, + 120, + 123, + 125, + 127, + 129, + 132, + 135, + 139, + 139, + 140, + 141, + 144, + 145, + 146, + 146, + 147, + 147, + 147, + 148, + 148, + 149, + 149, + 149, + 149 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 5, + 7, + 9, + 11, + 12, + 20, + 21, + 24, + 29, + 33, + 35, + 37, + 38, + 42, + 46, + 49, + 51, + 55, + 56, + 57, + 61, + 64, + 65, + 65, + 67, + 67, + 69, + 70, + 71, + 71, + 71, + 74, + 74, + 75, + 75, + 75, + 75, + 75 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82, + 0.84, + 0.86, + 0.88, + 0.9 + ] + ], + [ + 1593069993789.4, + 4, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3348214328289032, + 0.3363228738307953, + 0.3317972421646118, + 0.33816424012184143, + 0.3484848439693451, + 0.3492063581943512, + 0.3636363744735718, + 0.363095223903656, + 0.3717948794364929, + 0.37142857909202576, + 0.3840000033378601, + 0.3839285671710968, + 0.3979591727256775, + 0.4166666567325592, + 0.4722222089767456, + 0.460317462682724, + 0.4912280738353729, + 0.4901960790157318, + 0.5333333611488342, + 0.5641025900840759, + 0.5882353186607361, + 0.6666666865348816, + 0.625, + 0.5555555820465088, + 0.5625, + 0.6363636255264282, + 0.6666666865348816, + 0.6666666865348816, + 0.5, + 0.5, + 0.5, + 0.6666666865348816, + 1, + 1 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0.9599999785423279, + 0.9333333373069763, + 0.9200000166893005, + 0.8799999952316284, + 0.8533333539962769, + 0.8133333325386047, + 0.7733333110809326, + 0.6933333277702332, + 0.6399999856948853, + 0.5733333230018616, + 0.5199999809265137, + 0.46666666865348816, + 0.4533333480358124, + 0.3866666555404663, + 0.3733333349227905, + 0.3333333432674408, + 0.3199999928474426, + 0.2933333218097687, + 0.2666666805744171, + 0.23999999463558197, + 0.20000000298023224, + 0.13333334028720856, + 0.11999999731779099, + 0.09333333373069763, + 0.07999999821186066, + 0.0533333346247673, + 0.02666666731238365, + 0.02666666731238365, + 0.02666666731238365, + 0.02666666731238365, + 0.02666666731238365, + 0.013333333656191826 + ], + [ + 75, + 75, + 75, + 75, + 75, + 75, + 75, + 72, + 70, + 69, + 66, + 64, + 61, + 58, + 52, + 48, + 43, + 39, + 35, + 34, + 29, + 28, + 25, + 24, + 22, + 20, + 18, + 15, + 10, + 9, + 7, + 6, + 4, + 2, + 2, + 2, + 2, + 2, + 1 + ], + [ + 150, + 150, + 150, + 150, + 150, + 149, + 148, + 145, + 137, + 129, + 123, + 112, + 107, + 98, + 88, + 77, + 69, + 59, + 49, + 38, + 34, + 29, + 26, + 21, + 17, + 14, + 9, + 9, + 8, + 7, + 4, + 3, + 2, + 2, + 2, + 2, + 1, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 1, + 2, + 5, + 13, + 21, + 27, + 38, + 43, + 52, + 62, + 73, + 81, + 91, + 101, + 112, + 116, + 121, + 124, + 129, + 133, + 136, + 141, + 141, + 142, + 143, + 146, + 147, + 148, + 148, + 148, + 148, + 149, + 150, + 150 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 5, + 6, + 9, + 11, + 14, + 17, + 23, + 27, + 32, + 36, + 40, + 41, + 46, + 47, + 50, + 51, + 53, + 55, + 57, + 60, + 65, + 66, + 68, + 69, + 71, + 73, + 73, + 73, + 73, + 73, + 74 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78 + ] + ], + [ + 1593069993790.076, + 5, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3348214328289032, + 0.3378378450870514, + 0.3410138189792633, + 0.3507108986377716, + 0.3631840944290161, + 0.3617021143436432, + 0.3604651093482971, + 0.36250001192092896, + 0.3758389353752136, + 0.3834586441516876, + 0.4000000059604645, + 0.3980582654476166, + 0.44186046719551086, + 0.4430379867553711, + 0.48571428656578064, + 0.515625, + 0.5892857313156128, + 0.5490196347236633, + 0.5714285969734192, + 0.5641025900840759, + 0.5588235259056091, + 0.5862069129943848, + 0.5652173757553101, + 0.6000000238418579, + 0.6666666865348816, + 0.6470588445663452, + 0.6875, + 0.6666666865348816, + 0.6666666865348816, + 0.6666666865348816, + 0.75, + 0.7142857313156128, + 0.5, + 0.5, + 0.5, + 1 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0.9866666793823242, + 0.9866666793823242, + 0.9733333587646484, + 0.9066666960716248, + 0.8266666531562805, + 0.7733333110809326, + 0.746666669845581, + 0.6800000071525574, + 0.6133333444595337, + 0.54666668176651, + 0.5066666603088379, + 0.46666666865348816, + 0.4533333480358124, + 0.4399999976158142, + 0.4399999976158142, + 0.3733333349227905, + 0.3199999928474426, + 0.2933333218097687, + 0.25333333015441895, + 0.2266666740179062, + 0.1733333319425583, + 0.1599999964237213, + 0.1599999964237213, + 0.14666666090488434, + 0.14666666090488434, + 0.13333334028720856, + 0.1066666692495346, + 0.1066666692495346, + 0.07999999821186066, + 0.06666667014360428, + 0.02666666731238365, + 0.02666666731238365, + 0.013333333656191826, + 0.013333333656191826 + ], + [ + 75, + 75, + 75, + 75, + 75, + 75, + 75, + 74, + 74, + 73, + 68, + 62, + 58, + 56, + 51, + 46, + 41, + 38, + 35, + 34, + 33, + 33, + 28, + 24, + 22, + 19, + 17, + 13, + 12, + 12, + 11, + 11, + 10, + 8, + 8, + 6, + 5, + 2, + 2, + 1, + 1 + ], + [ + 150, + 150, + 150, + 150, + 150, + 149, + 147, + 143, + 137, + 128, + 120, + 110, + 102, + 93, + 82, + 69, + 62, + 48, + 44, + 36, + 31, + 23, + 23, + 18, + 17, + 15, + 12, + 10, + 8, + 6, + 6, + 5, + 5, + 4, + 4, + 2, + 2, + 2, + 2, + 1, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 1, + 3, + 7, + 13, + 22, + 30, + 40, + 48, + 57, + 68, + 81, + 88, + 102, + 106, + 114, + 119, + 127, + 127, + 132, + 133, + 135, + 138, + 140, + 142, + 144, + 144, + 145, + 145, + 146, + 146, + 148, + 148, + 148, + 148, + 149, + 150 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 2, + 7, + 13, + 17, + 19, + 24, + 29, + 34, + 37, + 40, + 41, + 42, + 42, + 47, + 51, + 53, + 56, + 58, + 62, + 63, + 63, + 64, + 64, + 65, + 67, + 67, + 69, + 70, + 73, + 73, + 74, + 74 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82 + ] + ], + [ + 1593069993790.763, + 6, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.33636364340782166, + 0.3396226465702057, + 0.34299516677856445, + 0.34825870394706726, + 0.35638296604156494, + 0.3641618490219116, + 0.3836477994918823, + 0.4041095972061157, + 0.4148148000240326, + 0.4406779706478119, + 0.4615384638309479, + 0.4285714328289032, + 0.44871795177459717, + 0.47826087474823, + 0.4769230782985687, + 0.4655172526836395, + 0.4901960790157318, + 0.54347825050354, + 0.523809552192688, + 0.5249999761581421, + 0.6000000238418579, + 0.5555555820465088, + 0.5416666865348816, + 0.5263158082962036, + 0.47058823704719543, + 0.3571428656578064, + 0.4545454680919647, + 0.6666666865348816, + 0.6000000238418579, + 0.5, + 0.5, + 0.6666666865348816, + 0.6666666865348816, + 0.6666666865348816, + 1, + 1, + 1, + 1, + 1 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 0.9866666793823242, + 0.9599999785423279, + 0.9466666579246521, + 0.9333333373069763, + 0.8933333158493042, + 0.8399999737739563, + 0.8133333325386047, + 0.7866666913032532, + 0.746666669845581, + 0.6933333277702332, + 0.6399999856948853, + 0.5199999809265137, + 0.46666666865348816, + 0.4399999976158142, + 0.41333332657814026, + 0.36000001430511475, + 0.3333333432674408, + 0.3333333432674408, + 0.2933333218097687, + 0.2800000011920929, + 0.2800000011920929, + 0.20000000298023224, + 0.1733333319425583, + 0.13333334028720856, + 0.1066666692495346, + 0.06666667014360428, + 0.06666667014360428, + 0.0533333346247673, + 0.03999999910593033, + 0.02666666731238365, + 0.02666666731238365, + 0.02666666731238365, + 0.02666666731238365, + 0.02666666731238365, + 0.02666666731238365, + 0.02666666731238365, + 0.02666666731238365, + 0.013333333656191826, + 0.013333333656191826 + ], + [ + 75, + 75, + 75, + 75, + 75, + 75, + 74, + 72, + 71, + 70, + 67, + 63, + 61, + 59, + 56, + 52, + 48, + 39, + 35, + 33, + 31, + 27, + 25, + 25, + 22, + 21, + 21, + 15, + 13, + 10, + 8, + 5, + 5, + 4, + 3, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 1, + 1 + ], + [ + 150, + 150, + 150, + 150, + 150, + 150, + 146, + 140, + 136, + 131, + 121, + 110, + 98, + 87, + 79, + 66, + 56, + 52, + 43, + 36, + 34, + 31, + 26, + 21, + 20, + 19, + 14, + 12, + 11, + 9, + 9, + 9, + 6, + 2, + 2, + 2, + 2, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 4, + 10, + 14, + 19, + 29, + 40, + 52, + 63, + 71, + 84, + 94, + 98, + 107, + 114, + 116, + 119, + 124, + 129, + 130, + 131, + 136, + 138, + 139, + 141, + 141, + 141, + 144, + 148, + 148, + 148, + 148, + 149, + 149, + 149, + 150, + 150, + 150, + 150, + 150 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 3, + 4, + 5, + 8, + 12, + 14, + 16, + 19, + 23, + 27, + 36, + 40, + 42, + 44, + 48, + 50, + 50, + 53, + 54, + 54, + 60, + 62, + 65, + 67, + 70, + 70, + 71, + 72, + 73, + 73, + 73, + 73, + 73, + 73, + 73, + 73, + 74, + 74 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82, + 0.84, + 0.86, + 0.88, + 0.9 + ] + ], + [ + 1593069993791.473, + 7, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3348214328289032, + 0.34246575832366943, + 0.3461538553237915, + 0.35567009449005127, + 0.36021506786346436, + 0.3735632300376892, + 0.3827160596847534, + 0.4054054021835327, + 0.3955223858356476, + 0.39830508828163147, + 0.4134615361690521, + 0.4204545319080353, + 0.44871795177459717, + 0.47887325286865234, + 0.5, + 0.5, + 0.5, + 0.5106382966041565, + 0.5365853905677795, + 0.5428571701049805, + 0.5625, + 0.5714285969734192, + 0.5769230723381042, + 0.5600000023841858, + 0.5454545617103577, + 0.5, + 0.5, + 0.5, + 0.4285714328289032, + 0.4545454680919647, + 0.4000000059604645, + 0.5, + 0.4000000059604645, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0.9599999785423279, + 0.9200000166893005, + 0.8933333158493042, + 0.8666666746139526, + 0.8266666531562805, + 0.800000011920929, + 0.7066666483879089, + 0.6266666650772095, + 0.5733333230018616, + 0.4933333396911621, + 0.46666666865348816, + 0.4533333480358124, + 0.4266666769981384, + 0.3733333349227905, + 0.3466666638851166, + 0.3199999928474426, + 0.2933333218097687, + 0.25333333015441895, + 0.23999999463558197, + 0.2133333384990692, + 0.20000000298023224, + 0.18666666746139526, + 0.1599999964237213, + 0.13333334028720856, + 0.11999999731779099, + 0.1066666692495346, + 0.07999999821186066, + 0.06666667014360428, + 0.0533333346247673, + 0.0533333346247673, + 0.02666666731238365, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 75, + 75, + 75, + 75, + 75, + 75, + 75, + 72, + 69, + 67, + 65, + 62, + 60, + 53, + 47, + 43, + 37, + 35, + 34, + 32, + 28, + 26, + 24, + 22, + 19, + 18, + 16, + 15, + 14, + 12, + 10, + 9, + 8, + 6, + 5, + 4, + 4, + 2, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 150, + 150, + 150, + 150, + 150, + 149, + 144, + 136, + 125, + 119, + 109, + 100, + 88, + 81, + 71, + 61, + 51, + 43, + 37, + 32, + 28, + 26, + 23, + 19, + 16, + 14, + 12, + 11, + 11, + 10, + 10, + 9, + 8, + 8, + 6, + 6, + 4, + 3, + 2, + 2, + 1, + 1, + 1, + 1 + ], + [ + 0, + 0, + 0, + 0, + 0, + 1, + 6, + 14, + 25, + 31, + 41, + 50, + 62, + 69, + 79, + 89, + 99, + 107, + 113, + 118, + 122, + 124, + 127, + 131, + 134, + 136, + 138, + 139, + 139, + 140, + 140, + 141, + 142, + 142, + 144, + 144, + 146, + 147, + 148, + 148, + 149, + 149, + 149, + 149 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 6, + 8, + 10, + 13, + 15, + 22, + 28, + 32, + 38, + 40, + 41, + 43, + 47, + 49, + 51, + 53, + 56, + 57, + 59, + 60, + 61, + 63, + 65, + 66, + 67, + 69, + 70, + 71, + 71, + 73, + 75, + 75, + 75, + 75, + 75, + 75 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82, + 0.84, + 0.86, + 0.88 + ] + ], + [ + 1593069993792.149, + 8, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3348214328289032, + 0.3363228738307953, + 0.3333333432674408, + 0.3396226465702057, + 0.3383084535598755, + 0.33673468232154846, + 0.3368983864784241, + 0.3314606845378876, + 0.33136093616485596, + 0.3561643958091736, + 0.3643410801887512, + 0.39316239953041077, + 0.43689319491386414, + 0.450549453496933, + 0.4938271641731262, + 0.4861111044883728, + 0.484375, + 0.5370370149612427, + 0.5333333611488342, + 0.5, + 0.5135135054588318, + 0.529411792755127, + 0.5806451439857483, + 0.6296296119689941, + 0.6499999761581421, + 0.6111111044883728, + 0.6666666865348816, + 0.7692307829856873, + 0.75, + 0.800000011920929, + 0.8333333134651184, + 0.800000011920929, + 0.800000011920929, + 1, + 1, + 1, + 1, + 1, + 1 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 0.9733333587646484, + 0.9599999785423279, + 0.9066666960716248, + 0.8799999952316284, + 0.8399999737739563, + 0.7866666913032532, + 0.746666669845581, + 0.6933333277702332, + 0.6266666650772095, + 0.6133333444595337, + 0.6000000238418579, + 0.54666668176651, + 0.5333333611488342, + 0.46666666865348816, + 0.41333332657814026, + 0.3866666555404663, + 0.3199999928474426, + 0.25333333015441895, + 0.25333333015441895, + 0.23999999463558197, + 0.23999999463558197, + 0.2266666740179062, + 0.1733333319425583, + 0.14666666090488434, + 0.13333334028720856, + 0.13333334028720856, + 0.11999999731779099, + 0.1066666692495346, + 0.06666667014360428, + 0.0533333346247673, + 0.0533333346247673, + 0.0533333346247673, + 0.03999999910593033, + 0.03999999910593033, + 0.03999999910593033, + 0.013333333656191826, + 0.013333333656191826 + ], + [ + 75, + 75, + 75, + 75, + 75, + 75, + 73, + 72, + 68, + 66, + 63, + 59, + 56, + 52, + 47, + 46, + 45, + 41, + 40, + 35, + 31, + 29, + 24, + 19, + 19, + 18, + 18, + 17, + 13, + 11, + 10, + 10, + 9, + 8, + 5, + 4, + 4, + 4, + 3, + 3, + 3, + 1, + 1 + ], + [ + 150, + 150, + 150, + 150, + 149, + 148, + 146, + 140, + 133, + 130, + 124, + 119, + 113, + 94, + 82, + 71, + 58, + 50, + 41, + 37, + 33, + 25, + 21, + 19, + 18, + 16, + 13, + 10, + 7, + 7, + 5, + 3, + 3, + 2, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 1, + 2, + 4, + 10, + 17, + 20, + 26, + 31, + 37, + 56, + 68, + 79, + 92, + 100, + 109, + 113, + 117, + 125, + 129, + 131, + 132, + 134, + 137, + 140, + 143, + 143, + 145, + 147, + 147, + 148, + 149, + 149, + 149, + 150, + 150, + 150, + 150, + 150, + 150 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 2, + 3, + 7, + 9, + 12, + 16, + 19, + 23, + 28, + 29, + 30, + 34, + 35, + 40, + 44, + 46, + 51, + 56, + 56, + 57, + 57, + 58, + 62, + 64, + 65, + 65, + 66, + 67, + 70, + 71, + 71, + 71, + 72, + 72, + 72, + 74, + 74 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82, + 0.84, + 0.86 + ] + ], + [ + 1593069993792.763, + 9, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3363228738307953, + 0.3395348787307739, + 0.3349282443523407, + 0.3283582031726837, + 0.32474225759506226, + 0.34682080149650574, + 0.3522012531757355, + 0.3611111044883728, + 0.36567163467407227, + 0.3781512677669525, + 0.37962964177131653, + 0.3958333432674408, + 0.42352941632270813, + 0.4399999976158142, + 0.4637681245803833, + 0.4754098355770111, + 0.46296295523643494, + 0.47058823704719543, + 0.4791666567325592, + 0.5128205418586731, + 0.5142857432365417, + 0.5161290168762207, + 0.5199999809265137, + 0.6000000238418579, + 0.5555555820465088, + 0.6666666865348816, + 0.7777777910232544, + 0.8571428656578064, + 0.8571428656578064, + 0.8571428656578064, + 0.8571428656578064, + 0.8333333134651184, + 0.8333333134651184, + 0.6666666865348816, + 0.6666666865348816, + 0 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 0.9733333587646484, + 0.9333333373069763, + 0.8799999952316284, + 0.8399999737739563, + 0.800000011920929, + 0.746666669845581, + 0.6933333277702332, + 0.653333306312561, + 0.6000000238418579, + 0.54666668176651, + 0.5066666603088379, + 0.47999998927116394, + 0.4399999976158142, + 0.4266666769981384, + 0.3866666555404663, + 0.3333333432674408, + 0.3199999928474426, + 0.30666667222976685, + 0.2666666805744171, + 0.23999999463558197, + 0.2133333384990692, + 0.1733333319425583, + 0.1599999964237213, + 0.13333334028720856, + 0.13333334028720856, + 0.09333333373069763, + 0.07999999821186066, + 0.07999999821186066, + 0.07999999821186066, + 0.07999999821186066, + 0.06666667014360428, + 0.06666667014360428, + 0.02666666731238365, + 0.02666666731238365, + 0 + ], + [ + 75, + 75, + 75, + 75, + 75, + 75, + 73, + 70, + 66, + 63, + 60, + 56, + 52, + 49, + 45, + 41, + 38, + 36, + 33, + 32, + 29, + 25, + 24, + 23, + 20, + 18, + 16, + 13, + 12, + 10, + 10, + 7, + 6, + 6, + 6, + 6, + 5, + 5, + 2, + 2, + 0 + ], + [ + 150, + 150, + 150, + 150, + 150, + 148, + 142, + 139, + 135, + 131, + 113, + 103, + 92, + 85, + 74, + 67, + 58, + 49, + 42, + 37, + 32, + 29, + 27, + 25, + 19, + 17, + 15, + 12, + 8, + 8, + 5, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + [ + 0, + 0, + 0, + 0, + 0, + 2, + 8, + 11, + 15, + 19, + 37, + 47, + 58, + 65, + 76, + 83, + 92, + 101, + 108, + 113, + 118, + 121, + 123, + 125, + 131, + 133, + 135, + 138, + 142, + 142, + 145, + 148, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + 149, + 149 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 2, + 5, + 9, + 12, + 15, + 19, + 23, + 26, + 30, + 34, + 37, + 39, + 42, + 43, + 46, + 50, + 51, + 52, + 55, + 57, + 59, + 62, + 63, + 65, + 65, + 68, + 69, + 69, + 69, + 69, + 70, + 70, + 73, + 73, + 75 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82 + ] + ] + ]; + } + return [ + [ + 1593069993538.6739, + 0, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3340757191181183, + 0.3340807259082794, + 0.3371298313140869, + 0.3372093141078949, + 0.35207822918891907, + 0.36269429326057434, + 0.3777777850627899, + 0.3857566714286804, + 0.39087948203086853, + 0.4021352231502533, + 0.4166666567325592, + 0.42489269375801086, + 0.44607841968536377, + 0.4748603403568268, + 0.4909090995788574, + 0.5104895234107971, + 0.48120301961898804, + 0.5041322112083435, + 0.5154638886451721, + 0.517241358757019, + 0.5263158082962036, + 0.6140350699424744, + 0.6595744490623474, + 0.675000011920929, + 0.6666666865348816, + 0.7727272510528564, + 0.7368420958518982, + 0.7058823704719543, + 0.7692307829856873, + 0.800000011920929, + 0.800000011920929, + 0.7777777910232544, + 0.75, + 0.7142857313156128, + 0.6000000238418579, + 0.5, + 0.5, + 0.5, + 0, + 0, + 0 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 0.9933333396911621, + 0.9866666793823242, + 0.9666666388511658, + 0.9599999785423279, + 0.9333333373069763, + 0.9066666960716248, + 0.8666666746139526, + 0.800000011920929, + 0.753333330154419, + 0.699999988079071, + 0.6600000262260437, + 0.6066666841506958, + 0.5666666626930237, + 0.5400000214576721, + 0.4866666793823242, + 0.4266666769981384, + 0.40666666626930237, + 0.3333333432674408, + 0.30000001192092896, + 0.2666666805744171, + 0.23333333432674408, + 0.20666666328907013, + 0.18000000715255737, + 0.13333334028720856, + 0.1133333370089531, + 0.09333333373069763, + 0.07999999821186066, + 0.06666667014360428, + 0.0533333346247673, + 0.0533333346247673, + 0.046666666865348816, + 0.03999999910593033, + 0.03333333507180214, + 0.019999999552965164, + 0.013333333656191826, + 0.006666666828095913, + 0.006666666828095913, + 0, + 0, + 0 + ], + [ + 150, + 150, + 150, + 150, + 150, + 150, + 149, + 148, + 145, + 144, + 140, + 136, + 130, + 120, + 113, + 105, + 99, + 91, + 85, + 81, + 73, + 64, + 61, + 50, + 45, + 40, + 35, + 31, + 27, + 20, + 17, + 14, + 12, + 10, + 8, + 8, + 7, + 6, + 5, + 3, + 2, + 1, + 1, + 0, + 0, + 0 + ], + [ + 300, + 300, + 300, + 300, + 300, + 299, + 297, + 291, + 285, + 265, + 246, + 224, + 207, + 187, + 168, + 147, + 134, + 113, + 94, + 84, + 70, + 69, + 60, + 47, + 42, + 36, + 22, + 16, + 13, + 10, + 5, + 5, + 5, + 3, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1 + ], + [ + 0, + 0, + 0, + 0, + 0, + 1, + 3, + 9, + 15, + 35, + 54, + 76, + 93, + 113, + 132, + 153, + 166, + 187, + 206, + 216, + 230, + 231, + 240, + 253, + 258, + 264, + 278, + 284, + 287, + 290, + 295, + 295, + 295, + 297, + 298, + 298, + 298, + 298, + 298, + 298, + 298, + 299, + 299, + 299, + 299, + 299 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 2, + 5, + 6, + 10, + 14, + 20, + 30, + 37, + 45, + 51, + 59, + 65, + 69, + 77, + 86, + 89, + 100, + 105, + 110, + 115, + 119, + 123, + 130, + 133, + 136, + 138, + 140, + 142, + 142, + 143, + 144, + 145, + 147, + 148, + 149, + 149, + 150, + 150, + 150 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82, + 0.84, + 0.86, + 0.88, + 0.9, + 0.92 + ] + ], + [ + 1593069993539.396, + 1, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.33557048439979553, + 0.33560091257095337, + 0.3498817980289459, + 0.35853657126426697, + 0.37402597069740295, + 0.3835616409778595, + 0.3797101378440857, + 0.3967213034629822, + 0.40072202682495117, + 0.40816327929496765, + 0.4139534831047058, + 0.4334975481033325, + 0.45652174949645996, + 0.4848484992980957, + 0.4829931855201721, + 0.5, + 0.49180328845977783, + 0.49074074625968933, + 0.52173912525177, + 0.5443037748336792, + 0.5352112650871277, + 0.5245901346206665, + 0.4901960790157318, + 0.4545454680919647, + 0.4324324429035187, + 0.4375, + 0.4399999976158142, + 0.5, + 0.5625, + 0.5, + 0.4000000059604645, + 0.5, + 0.4000000059604645, + 0.5, + 0, + 0, + 0 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0.9866666793823242, + 0.9866666793823242, + 0.9800000190734863, + 0.9599999785423279, + 0.9333333373069763, + 0.8733333349227905, + 0.8066666722297668, + 0.7400000095367432, + 0.6666666865348816, + 0.5933333039283752, + 0.5866666436195374, + 0.5600000023841858, + 0.5333333611488342, + 0.47333332896232605, + 0.46000000834465027, + 0.4000000059604645, + 0.35333332419395447, + 0.3199999928474426, + 0.2866666615009308, + 0.25333333015441895, + 0.2133333384990692, + 0.1666666716337204, + 0.13333334028720856, + 0.1066666692495346, + 0.09333333373069763, + 0.07333333045244217, + 0.06666667014360428, + 0.05999999865889549, + 0.03999999910593033, + 0.02666666731238365, + 0.02666666731238365, + 0.013333333656191826, + 0.006666666828095913, + 0, + 0, + 0 + ], + [ + 150, + 150, + 150, + 150, + 150, + 150, + 150, + 148, + 148, + 147, + 144, + 140, + 131, + 121, + 111, + 100, + 89, + 88, + 84, + 80, + 71, + 69, + 60, + 53, + 48, + 43, + 38, + 32, + 25, + 20, + 16, + 14, + 11, + 10, + 9, + 6, + 4, + 4, + 2, + 1, + 0, + 0, + 0 + ], + [ + 300, + 300, + 300, + 300, + 300, + 300, + 297, + 293, + 275, + 263, + 241, + 225, + 214, + 184, + 166, + 145, + 126, + 115, + 100, + 85, + 76, + 69, + 62, + 55, + 44, + 36, + 33, + 29, + 26, + 24, + 21, + 18, + 14, + 10, + 7, + 6, + 6, + 4, + 3, + 1, + 1, + 1, + 1 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 3, + 7, + 25, + 37, + 59, + 75, + 86, + 116, + 134, + 155, + 174, + 185, + 200, + 215, + 224, + 231, + 238, + 245, + 256, + 264, + 267, + 271, + 274, + 276, + 279, + 282, + 286, + 290, + 293, + 294, + 294, + 296, + 297, + 299, + 299, + 299, + 299 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2, + 2, + 3, + 6, + 10, + 19, + 29, + 39, + 50, + 61, + 62, + 66, + 70, + 79, + 81, + 90, + 97, + 102, + 107, + 112, + 118, + 125, + 130, + 134, + 136, + 139, + 140, + 141, + 144, + 146, + 146, + 148, + 149, + 150, + 150, + 150 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82, + 0.84, + 0.86 + ] + ], + [ + 1593069993540.066, + 2, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3348214328289032, + 0.33484163880348206, + 0.33870968222618103, + 0.34433960914611816, + 0.35749998688697815, + 0.3658536672592163, + 0.3742690086364746, + 0.3853503167629242, + 0.4027777910232544, + 0.40784314274787903, + 0.4025973975658417, + 0.4166666567325592, + 0.42328041791915894, + 0.45783132314682007, + 0.4751773178577423, + 0.4296875, + 0.4568965435028076, + 0.4476190507411957, + 0.4838709533214569, + 0.5121951103210449, + 0.4931506812572479, + 0.508474588394165, + 0.4901960790157318, + 0.4888888895511627, + 0.5555555820465088, + 0.6333333253860474, + 0.6296296119689941, + 0.75, + 0.75, + 0.6666666865348816, + 0.75, + 0.7142857313156128, + 0.6666666865348816, + 0.3333333432674408, + 0, + 0 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 0.9866666793823242, + 0.9800000190734863, + 0.9733333587646484, + 0.95333331823349, + 0.8999999761581421, + 0.8533333539962769, + 0.8066666722297668, + 0.7733333110809326, + 0.6933333277702332, + 0.6200000047683716, + 0.5666666626930237, + 0.5333333611488342, + 0.5066666603088379, + 0.4466666579246521, + 0.36666667461395264, + 0.35333332419395447, + 0.31333333253860474, + 0.30000001192092896, + 0.2800000011920929, + 0.23999999463558197, + 0.20000000298023224, + 0.1666666716337204, + 0.14666666090488434, + 0.13333334028720856, + 0.12666666507720947, + 0.1133333370089531, + 0.10000000149011612, + 0.07999999821186066, + 0.0533333346247673, + 0.03999999910593033, + 0.03333333507180214, + 0.02666666731238365, + 0.006666666828095913, + 0, + 0 + ], + [ + 150, + 150, + 150, + 150, + 150, + 150, + 148, + 147, + 146, + 143, + 135, + 128, + 121, + 116, + 104, + 93, + 85, + 80, + 76, + 67, + 55, + 53, + 47, + 45, + 42, + 36, + 30, + 25, + 22, + 20, + 19, + 17, + 15, + 12, + 8, + 6, + 5, + 4, + 1, + 0, + 0 + ], + [ + 300, + 300, + 300, + 300, + 300, + 298, + 294, + 287, + 278, + 257, + 234, + 214, + 193, + 172, + 151, + 138, + 119, + 109, + 90, + 74, + 73, + 63, + 58, + 48, + 40, + 37, + 29, + 26, + 23, + 16, + 11, + 10, + 5, + 4, + 4, + 2, + 2, + 2, + 2, + 1, + 1 + ], + [ + 0, + 0, + 0, + 0, + 0, + 2, + 6, + 13, + 22, + 43, + 66, + 86, + 107, + 128, + 149, + 162, + 181, + 191, + 210, + 226, + 227, + 237, + 242, + 252, + 260, + 263, + 271, + 274, + 277, + 284, + 289, + 290, + 295, + 296, + 296, + 298, + 298, + 298, + 298, + 299, + 299 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 2, + 3, + 4, + 7, + 15, + 22, + 29, + 34, + 46, + 57, + 65, + 70, + 74, + 83, + 95, + 97, + 103, + 105, + 108, + 114, + 120, + 125, + 128, + 130, + 131, + 133, + 135, + 138, + 142, + 144, + 145, + 146, + 149, + 150, + 150 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82 + ] + ], + [ + 1593069993540.662, + 3, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.33178654313087463, + 0.33571428060531616, + 0.3350125849246979, + 0.3448275923728943, + 0.36467236280441284, + 0.3639143705368042, + 0.3877550959587097, + 0.38661709427833557, + 0.38211381435394287, + 0.3963963985443115, + 0.4108910858631134, + 0.43406593799591064, + 0.4491018056869507, + 0.4652777910232544, + 0.4880000054836273, + 0.4955752193927765, + 0.5145630836486816, + 0.5, + 0.5057471394538879, + 0.5194805264472961, + 0.5373134613037109, + 0.5789473652839661, + 0.6000000238418579, + 0.5952380895614624, + 0.5714285969734192, + 0.6785714030265808, + 0.6521739363670349, + 0.6666666865348816, + 0.7142857313156128, + 0.7272727489471436, + 0.6666666865348816, + 0.5, + 0.6000000238418579, + 0.5, + 0.6666666865348816, + 0.6666666865348816, + 0.6666666865348816, + 0.5, + 1 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 0.9866666793823242, + 0.95333331823349, + 0.9399999976158142, + 0.8866666555404663, + 0.8666666746139526, + 0.8533333539962769, + 0.7933333516120911, + 0.7599999904632568, + 0.6933333277702332, + 0.6266666650772095, + 0.5866666436195374, + 0.5533333420753479, + 0.5266666412353516, + 0.5, + 0.4466666579246521, + 0.40666666626930237, + 0.3733333349227905, + 0.35333332419395447, + 0.31333333253860474, + 0.2933333218097687, + 0.2666666805744171, + 0.23999999463558197, + 0.2199999988079071, + 0.20000000298023224, + 0.1666666716337204, + 0.13333334028720856, + 0.12666666507720947, + 0.10000000149011612, + 0.07999999821186066, + 0.06666667014360428, + 0.0533333346247673, + 0.03999999910593033, + 0.019999999552965164, + 0.019999999552965164, + 0.013333333656191826, + 0.013333333656191826, + 0.013333333656191826, + 0.013333333656191826, + 0.006666666828095913, + 0.006666666828095913 + ], + [ + 150, + 150, + 150, + 150, + 150, + 150, + 148, + 143, + 141, + 133, + 130, + 128, + 119, + 114, + 104, + 94, + 88, + 83, + 79, + 75, + 67, + 61, + 56, + 53, + 47, + 44, + 40, + 36, + 33, + 30, + 25, + 20, + 19, + 15, + 12, + 10, + 8, + 6, + 3, + 3, + 2, + 2, + 2, + 2, + 1, + 1 + ], + [ + 300, + 300, + 300, + 300, + 300, + 300, + 296, + 288, + 279, + 264, + 247, + 223, + 208, + 180, + 165, + 152, + 134, + 119, + 103, + 92, + 77, + 64, + 57, + 50, + 47, + 43, + 37, + 31, + 24, + 20, + 17, + 15, + 9, + 8, + 6, + 4, + 3, + 3, + 3, + 2, + 2, + 1, + 1, + 1, + 1, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 4, + 12, + 21, + 36, + 53, + 77, + 92, + 120, + 135, + 148, + 166, + 181, + 197, + 208, + 223, + 236, + 243, + 250, + 253, + 257, + 263, + 269, + 276, + 280, + 283, + 285, + 291, + 292, + 294, + 296, + 297, + 297, + 297, + 298, + 298, + 299, + 299, + 299, + 299, + 300 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 2, + 7, + 9, + 17, + 20, + 22, + 31, + 36, + 46, + 56, + 62, + 67, + 71, + 75, + 83, + 89, + 94, + 97, + 103, + 106, + 110, + 114, + 117, + 120, + 125, + 130, + 131, + 135, + 138, + 140, + 142, + 144, + 147, + 147, + 148, + 148, + 148, + 148, + 149, + 149 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82, + 0.84, + 0.86, + 0.88, + 0.9, + 0.92 + ] + ], + [ + 1593069993541.333, + 4, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3340757191181183, + 0.3363228738307953, + 0.33563217520713806, + 0.3445783257484436, + 0.3544303774833679, + 0.36164382100105286, + 0.366568922996521, + 0.369905948638916, + 0.37931033968925476, + 0.3807692229747772, + 0.40611353516578674, + 0.41747573018074036, + 0.44324323534965515, + 0.4522292912006378, + 0.48905110359191895, + 0.4880000054836273, + 0.5185185074806213, + 0.5257731676101685, + 0.5454545617103577, + 0.5512820482254028, + 0.5735294222831726, + 0.5964912176132202, + 0.5961538553237915, + 0.5609756112098694, + 0.5675675868988037, + 0.6428571343421936, + 0.6363636255264282, + 0.6470588445663452, + 0.5, + 0.5454545617103577, + 0.5555555820465088, + 0.625, + 0.6666666865348816, + 0.3333333432674408, + 0, + 0, + 0, + 0 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0.9733333587646484, + 0.95333331823349, + 0.9333333373069763, + 0.8799999952316284, + 0.8333333134651184, + 0.7866666913032532, + 0.7333333492279053, + 0.6600000262260437, + 0.6200000047683716, + 0.5733333230018616, + 0.54666668176651, + 0.47333332896232605, + 0.4466666579246521, + 0.40666666626930237, + 0.3733333349227905, + 0.3400000035762787, + 0.3199999928474426, + 0.2866666615009308, + 0.25999999046325684, + 0.2266666740179062, + 0.20666666328907013, + 0.15333333611488342, + 0.14000000059604645, + 0.11999999731779099, + 0.09333333373069763, + 0.07333333045244217, + 0.03999999910593033, + 0.03999999910593033, + 0.03333333507180214, + 0.03333333507180214, + 0.02666666731238365, + 0.006666666828095913, + 0, + 0, + 0, + 0 + ], + [ + 150, + 150, + 150, + 150, + 150, + 150, + 150, + 146, + 143, + 140, + 132, + 125, + 118, + 110, + 99, + 93, + 86, + 82, + 71, + 67, + 61, + 56, + 51, + 48, + 43, + 39, + 34, + 31, + 23, + 21, + 18, + 14, + 11, + 6, + 6, + 5, + 5, + 4, + 1, + 0, + 0, + 0, + 0 + ], + [ + 300, + 300, + 300, + 300, + 300, + 299, + 296, + 289, + 272, + 255, + 233, + 216, + 201, + 180, + 161, + 136, + 120, + 103, + 86, + 70, + 64, + 52, + 46, + 40, + 35, + 29, + 23, + 21, + 18, + 16, + 10, + 8, + 6, + 6, + 5, + 4, + 3, + 2, + 2, + 2, + 2, + 1, + 1 + ], + [ + 0, + 0, + 0, + 0, + 0, + 1, + 4, + 11, + 28, + 45, + 67, + 84, + 99, + 120, + 139, + 164, + 180, + 197, + 214, + 230, + 236, + 248, + 254, + 260, + 265, + 271, + 277, + 279, + 282, + 284, + 290, + 292, + 294, + 294, + 295, + 296, + 297, + 298, + 298, + 298, + 298, + 299, + 299 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 4, + 7, + 10, + 18, + 25, + 32, + 40, + 51, + 57, + 64, + 68, + 79, + 83, + 89, + 94, + 99, + 102, + 107, + 111, + 116, + 119, + 127, + 129, + 132, + 136, + 139, + 144, + 144, + 145, + 145, + 146, + 149, + 150, + 150, + 150, + 150 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82, + 0.84, + 0.86 + ] + ], + [ + 1593069993542.078, + 5, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.33557048439979553, + 0.3355855941772461, + 0.33944955468177795, + 0.3501199185848236, + 0.35249999165534973, + 0.35978835821151733, + 0.35919541120529175, + 0.35403725504875183, + 0.37288135290145874, + 0.3730769157409668, + 0.3822222352027893, + 0.39086294174194336, + 0.42941176891326904, + 0.4394904375076294, + 0.47183099389076233, + 0.5079365372657776, + 0.5398229956626892, + 0.5148515105247498, + 0.5348837375640869, + 0.5432098507881165, + 0.5882353186607361, + 0.5932203531265259, + 0.5714285969734192, + 0.5909090638160706, + 0.6341463327407837, + 0.6285714507102966, + 0.6333333253860474, + 0.6000000238418579, + 0.5714285969734192, + 0.5263158082962036, + 0.5, + 0.4166666567325592, + 0.2222222238779068, + 0.2857142984867096, + 0.25, + 0.3333333432674408, + 0, + 0, + 0 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 0.9933333396911621, + 0.9866666793823242, + 0.9733333587646484, + 0.9399999976158142, + 0.9066666960716248, + 0.8333333134651184, + 0.7599999904632568, + 0.7333333492279053, + 0.6466666460037231, + 0.5733333230018616, + 0.5133333206176758, + 0.4866666793823242, + 0.46000000834465027, + 0.4466666579246521, + 0.4266666769981384, + 0.40666666626930237, + 0.3466666638851166, + 0.30666667222976685, + 0.2933333218097687, + 0.2666666805744171, + 0.23333333432674408, + 0.18666666746139526, + 0.1733333319425583, + 0.1733333319425583, + 0.14666666090488434, + 0.12666666507720947, + 0.10000000149011612, + 0.07999999821186066, + 0.06666667014360428, + 0.046666666865348816, + 0.03333333507180214, + 0.013333333656191826, + 0.013333333656191826, + 0.006666666828095913, + 0.006666666828095913, + 0, + 0, + 0 + ], + [ + 150, + 150, + 150, + 150, + 150, + 150, + 149, + 148, + 146, + 141, + 136, + 125, + 114, + 110, + 97, + 86, + 77, + 73, + 69, + 67, + 64, + 61, + 52, + 46, + 44, + 40, + 35, + 28, + 26, + 26, + 22, + 19, + 15, + 12, + 10, + 7, + 5, + 2, + 2, + 1, + 1, + 0, + 0, + 0 + ], + [ + 300, + 300, + 300, + 300, + 300, + 297, + 295, + 288, + 271, + 259, + 242, + 223, + 208, + 185, + 163, + 139, + 120, + 97, + 88, + 75, + 62, + 52, + 49, + 40, + 37, + 28, + 24, + 21, + 18, + 15, + 13, + 11, + 10, + 9, + 9, + 7, + 7, + 7, + 5, + 3, + 2, + 2, + 1, + 1 + ], + [ + 0, + 0, + 0, + 0, + 0, + 3, + 5, + 12, + 29, + 41, + 58, + 77, + 92, + 115, + 137, + 161, + 180, + 203, + 212, + 225, + 238, + 248, + 251, + 260, + 263, + 272, + 276, + 279, + 282, + 285, + 287, + 289, + 290, + 291, + 291, + 293, + 293, + 293, + 295, + 297, + 298, + 298, + 299, + 299 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 2, + 4, + 9, + 14, + 25, + 36, + 40, + 53, + 64, + 73, + 77, + 81, + 83, + 86, + 89, + 98, + 104, + 106, + 110, + 115, + 122, + 124, + 124, + 128, + 131, + 135, + 138, + 140, + 143, + 145, + 148, + 148, + 149, + 149, + 150, + 150, + 150 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82, + 0.84, + 0.86, + 0.88 + ] + ], + [ + 1593069993543.1821, + 6, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3303571343421936, + 0.33257919549942017, + 0.33642691373825073, + 0.34285715222358704, + 0.35087719559669495, + 0.36239781975746155, + 0.37134504318237305, + 0.38485804200172424, + 0.4000000059604645, + 0.4089219272136688, + 0.43096235394477844, + 0.43518519401550293, + 0.4202127754688263, + 0.42603549361228943, + 0.43918919563293457, + 0.44525548815727234, + 0.45967742800712585, + 0.4821428656578064, + 0.5102040767669678, + 0.529411792755127, + 0.5128205418586731, + 0.5797101259231567, + 0.550000011920929, + 0.5094339847564697, + 0.4888888895511627, + 0.46341463923454285, + 0.375, + 0.3461538553237915, + 0.4000000059604645, + 0.4000000059604645, + 0.3333333432674408, + 0.27272728085517883, + 0.3333333432674408, + 0.3333333432674408, + 0.2857142984867096, + 0.3333333432674408, + 0.3333333432674408, + 0.6666666865348816, + 0.5, + 1 + ], + [ + 1, + 1, + 1, + 1, + 1, + 0.9866666793823242, + 0.9800000190734863, + 0.9666666388511658, + 0.9599999785423279, + 0.9333333373069763, + 0.8866666555404663, + 0.846666693687439, + 0.8133333325386047, + 0.7599999904632568, + 0.7333333492279053, + 0.6866666674613953, + 0.6266666650772095, + 0.5266666412353516, + 0.47999998927116394, + 0.4333333373069763, + 0.40666666626930237, + 0.3799999952316284, + 0.36000001430511475, + 0.3333333432674408, + 0.30000001192092896, + 0.2666666805744171, + 0.2666666805744171, + 0.2199999988079071, + 0.18000000715255737, + 0.14666666090488434, + 0.12666666507720947, + 0.07999999821186066, + 0.05999999865889549, + 0.0533333346247673, + 0.03999999910593033, + 0.02666666731238365, + 0.019999999552965164, + 0.019999999552965164, + 0.019999999552965164, + 0.013333333656191826, + 0.013333333656191826, + 0.013333333656191826, + 0.013333333656191826, + 0.006666666828095913, + 0.006666666828095913 + ], + [ + 150, + 150, + 150, + 150, + 150, + 148, + 147, + 145, + 144, + 140, + 133, + 127, + 122, + 114, + 110, + 103, + 94, + 79, + 72, + 65, + 61, + 57, + 54, + 50, + 45, + 40, + 40, + 33, + 27, + 22, + 19, + 12, + 9, + 8, + 6, + 4, + 3, + 3, + 3, + 2, + 2, + 2, + 2, + 1, + 1 + ], + [ + 300, + 300, + 300, + 300, + 300, + 300, + 295, + 286, + 276, + 259, + 234, + 215, + 195, + 171, + 159, + 136, + 122, + 109, + 97, + 83, + 76, + 67, + 58, + 48, + 40, + 38, + 29, + 27, + 26, + 23, + 22, + 20, + 17, + 12, + 9, + 8, + 8, + 6, + 6, + 5, + 4, + 4, + 1, + 1, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 5, + 14, + 24, + 41, + 66, + 85, + 105, + 129, + 141, + 164, + 178, + 191, + 203, + 217, + 224, + 233, + 242, + 252, + 260, + 262, + 271, + 273, + 274, + 277, + 278, + 280, + 283, + 288, + 291, + 292, + 292, + 294, + 294, + 295, + 296, + 296, + 299, + 299, + 300 + ], + [ + 0, + 0, + 0, + 0, + 0, + 2, + 3, + 5, + 6, + 10, + 17, + 23, + 28, + 36, + 40, + 47, + 56, + 71, + 78, + 85, + 89, + 93, + 96, + 100, + 105, + 110, + 110, + 117, + 123, + 128, + 131, + 138, + 141, + 142, + 144, + 146, + 147, + 147, + 147, + 148, + 148, + 148, + 148, + 149, + 149 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82, + 0.84, + 0.86, + 0.88, + 0.9 + ] + ], + [ + 1593069993543.998, + 7, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3340757191181183, + 0.3348214328289032, + 0.33636364340782166, + 0.3380281627178192, + 0.3483709394931793, + 0.35449734330177307, + 0.3672316372394562, + 0.381538450717926, + 0.39464882016181946, + 0.3970588147640228, + 0.40416666865348816, + 0.4084506928920746, + 0.42328041791915894, + 0.4497041404247284, + 0.4868420958518982, + 0.5072463750839233, + 0.529411792755127, + 0.5267857313156128, + 0.5050504803657532, + 0.5056179761886597, + 0.5131579041481018, + 0.5072463750839233, + 0.515625, + 0.5636363625526428, + 0.5686274766921997, + 0.5744680762290955, + 0.5581395626068115, + 0.5277777910232544, + 0.5, + 0.43478259444236755, + 0.4444444477558136, + 0.4375, + 0.5833333134651184, + 0.5, + 0.25, + 0, + 0, + 0, + 0, + 0 + ], + [ + 1, + 1, + 1, + 1, + 1, + 1, + 0.9866666793823242, + 0.9599999785423279, + 0.9266666769981384, + 0.8933333158493042, + 0.8666666746139526, + 0.8266666531562805, + 0.7866666913032532, + 0.7200000286102295, + 0.6466666460037231, + 0.5799999833106995, + 0.5333333611488342, + 0.5066666603088379, + 0.4933333396911621, + 0.46666666865348816, + 0.41999998688697815, + 0.3933333456516266, + 0.3333333432674408, + 0.30000001192092896, + 0.25999999046325684, + 0.23333333432674408, + 0.2199999988079071, + 0.20666666328907013, + 0.19333332777023315, + 0.18000000715255737, + 0.1599999964237213, + 0.12666666507720947, + 0.10000000149011612, + 0.06666667014360428, + 0.0533333346247673, + 0.046666666865348816, + 0.046666666865348816, + 0.02666666731238365, + 0.006666666828095913, + 0, + 0, + 0, + 0, + 0 + ], + [ + 150, + 150, + 150, + 150, + 150, + 150, + 148, + 144, + 139, + 134, + 130, + 124, + 118, + 108, + 97, + 87, + 80, + 76, + 74, + 70, + 63, + 59, + 50, + 45, + 39, + 35, + 33, + 31, + 29, + 27, + 24, + 19, + 15, + 10, + 8, + 7, + 7, + 4, + 1, + 0, + 0, + 0, + 0, + 0 + ], + [ + 300, + 300, + 300, + 300, + 299, + 298, + 292, + 282, + 260, + 244, + 224, + 201, + 181, + 164, + 143, + 126, + 109, + 93, + 78, + 68, + 56, + 53, + 49, + 44, + 37, + 34, + 31, + 24, + 22, + 20, + 19, + 17, + 15, + 13, + 10, + 9, + 5, + 4, + 3, + 2, + 1, + 1, + 1, + 1 + ], + [ + 0, + 0, + 0, + 0, + 1, + 2, + 8, + 18, + 40, + 56, + 76, + 99, + 119, + 136, + 157, + 174, + 191, + 207, + 222, + 232, + 244, + 247, + 251, + 256, + 263, + 266, + 269, + 276, + 278, + 280, + 281, + 283, + 285, + 287, + 290, + 291, + 295, + 296, + 297, + 298, + 299, + 299, + 299, + 299 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 2, + 6, + 11, + 16, + 20, + 26, + 32, + 42, + 53, + 63, + 70, + 74, + 76, + 80, + 87, + 91, + 100, + 105, + 111, + 115, + 117, + 119, + 121, + 123, + 126, + 131, + 135, + 140, + 142, + 143, + 143, + 146, + 149, + 150, + 150, + 150, + 150, + 150 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82, + 0.84, + 0.86, + 0.88 + ] + ], + [ + 1593069993544.9128, + 8, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3340757191181183, + 0.33483144640922546, + 0.33638444542884827, + 0.3452380895614624, + 0.34405940771102905, + 0.3489583432674408, + 0.35164836049079895, + 0.34593021869659424, + 0.35825544595718384, + 0.38327527046203613, + 0.3897637724876404, + 0.4196428656578064, + 0.44607841968536377, + 0.45945945382118225, + 0.4819277226924896, + 0.4791666567325592, + 0.484375, + 0.5044247508049011, + 0.5102040767669678, + 0.5176470875740051, + 0.5375000238418579, + 0.5540540814399719, + 0.5671641826629639, + 0.5833333134651184, + 0.6170212626457214, + 0.625, + 0.7096773982048035, + 0.7307692170143127, + 0.7727272510528564, + 0.8333333134651184, + 0.8571428656578064, + 0.8181818127632141, + 0.8999999761581421, + 1, + 1, + 1, + 1, + 1, + 1 + ], + [ + 1, + 1, + 1, + 1, + 1, + 0.9933333396911621, + 0.9800000190734863, + 0.9666666388511658, + 0.9266666769981384, + 0.8933333158493042, + 0.8533333539962769, + 0.7933333516120911, + 0.7666666507720947, + 0.7333333492279053, + 0.6600000262260437, + 0.6266666650772095, + 0.6066666841506958, + 0.5666666626930237, + 0.5333333611488342, + 0.46000000834465027, + 0.41333332657814026, + 0.3799999952316284, + 0.3333333432674408, + 0.2933333218097687, + 0.2866666615009308, + 0.273333340883255, + 0.25333333015441895, + 0.23333333432674408, + 0.19333332777023315, + 0.1666666716337204, + 0.14666666090488434, + 0.12666666507720947, + 0.1133333370089531, + 0.10000000149011612, + 0.07999999821186066, + 0.05999999865889549, + 0.05999999865889549, + 0.046666666865348816, + 0.03999999910593033, + 0.03999999910593033, + 0.03333333507180214, + 0.013333333656191826, + 0.013333333656191826 + ], + [ + 150, + 150, + 150, + 150, + 150, + 149, + 147, + 145, + 139, + 134, + 128, + 119, + 115, + 110, + 99, + 94, + 91, + 85, + 80, + 69, + 62, + 57, + 50, + 44, + 43, + 41, + 38, + 35, + 29, + 25, + 22, + 19, + 17, + 15, + 12, + 9, + 9, + 7, + 6, + 6, + 5, + 2, + 2 + ], + [ + 300, + 300, + 300, + 300, + 299, + 296, + 290, + 275, + 265, + 250, + 236, + 225, + 206, + 177, + 155, + 130, + 113, + 100, + 86, + 75, + 66, + 56, + 48, + 41, + 37, + 33, + 29, + 25, + 18, + 15, + 9, + 7, + 5, + 3, + 2, + 2, + 1, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 1, + 4, + 10, + 25, + 35, + 50, + 64, + 75, + 94, + 123, + 145, + 170, + 187, + 200, + 214, + 225, + 234, + 244, + 252, + 259, + 263, + 267, + 271, + 275, + 282, + 285, + 291, + 293, + 295, + 297, + 298, + 298, + 299, + 300, + 300, + 300, + 300, + 300, + 300 + ], + [ + 0, + 0, + 0, + 0, + 0, + 1, + 3, + 5, + 11, + 16, + 22, + 31, + 35, + 40, + 51, + 56, + 59, + 65, + 70, + 81, + 88, + 93, + 100, + 106, + 107, + 109, + 112, + 115, + 121, + 125, + 128, + 131, + 133, + 135, + 138, + 141, + 141, + 143, + 144, + 144, + 145, + 148, + 148 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82, + 0.84, + 0.86 + ] + ], + [ + 1593069993545.62, + 9, + [ + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3333333432674408, + 0.3340807259082794, + 0.33870968222618103, + 0.3388625681400299, + 0.3432098627090454, + 0.35078534483909607, + 0.37321937084198, + 0.3792048990726471, + 0.3872053921222687, + 0.3955223858356476, + 0.42016807198524475, + 0.43317973613739014, + 0.43617022037506104, + 0.4588235318660736, + 0.4736842215061188, + 0.4848484992980957, + 0.5, + 0.49038460850715637, + 0.5106382966041565, + 0.5411764979362488, + 0.5857142806053162, + 0.5645161271095276, + 0.5714285969734192, + 0.6041666865348816, + 0.6666666865348816, + 0.6571428775787354, + 0.7333333492279053, + 0.8260869383811951, + 0.8333333134651184, + 0.875, + 0.8181818127632141, + 0.8181818127632141, + 0.8888888955116272, + 0.8888888955116272, + 0.800000011920929, + 0.800000011920929, + 0 + ], + [ + 1, + 1, + 1, + 1, + 1, + 0.9933333396911621, + 0.9800000190734863, + 0.95333331823349, + 0.9266666769981384, + 0.8933333158493042, + 0.8733333349227905, + 0.8266666531562805, + 0.7666666507720947, + 0.7066666483879089, + 0.6666666865348816, + 0.6266666650772095, + 0.54666668176651, + 0.5199999809265137, + 0.47999998927116394, + 0.4266666769981384, + 0.3933333456516266, + 0.3400000035762787, + 0.3199999928474426, + 0.30666667222976685, + 0.273333340883255, + 0.23333333432674408, + 0.2133333384990692, + 0.19333332777023315, + 0.1733333319425583, + 0.15333333611488342, + 0.14666666090488434, + 0.12666666507720947, + 0.10000000149011612, + 0.09333333373069763, + 0.05999999865889549, + 0.05999999865889549, + 0.0533333346247673, + 0.0533333346247673, + 0.02666666731238365, + 0.02666666731238365, + 0 + ], + [ + 150, + 150, + 150, + 150, + 150, + 149, + 147, + 143, + 139, + 134, + 131, + 124, + 115, + 106, + 100, + 94, + 82, + 78, + 72, + 64, + 59, + 51, + 48, + 46, + 41, + 35, + 32, + 29, + 26, + 23, + 22, + 19, + 15, + 14, + 9, + 9, + 8, + 8, + 4, + 4, + 0 + ], + [ + 300, + 300, + 300, + 300, + 300, + 297, + 287, + 279, + 266, + 248, + 220, + 203, + 182, + 162, + 138, + 123, + 106, + 92, + 80, + 68, + 59, + 53, + 46, + 39, + 29, + 27, + 24, + 19, + 13, + 12, + 8, + 4, + 3, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1 + ], + [ + 0, + 0, + 0, + 0, + 0, + 3, + 13, + 21, + 34, + 52, + 80, + 97, + 118, + 138, + 162, + 177, + 194, + 208, + 220, + 232, + 241, + 247, + 254, + 261, + 271, + 273, + 276, + 281, + 287, + 288, + 292, + 296, + 297, + 298, + 298, + 298, + 299, + 299, + 299, + 299, + 299 + ], + [ + 0, + 0, + 0, + 0, + 0, + 1, + 3, + 7, + 11, + 16, + 19, + 26, + 35, + 44, + 50, + 56, + 68, + 72, + 78, + 86, + 91, + 99, + 102, + 104, + 109, + 115, + 118, + 121, + 124, + 127, + 128, + 131, + 135, + 136, + 141, + 141, + 142, + 142, + 146, + 146, + 150 + ], + [ + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.12, + 0.14, + 0.16, + 0.18, + 0.2, + 0.22, + 0.24, + 0.26, + 0.28, + 0.3, + 0.32, + 0.34, + 0.36, + 0.38, + 0.4, + 0.42, + 0.44, + 0.46, + 0.48, + 0.5, + 0.52, + 0.54, + 0.56, + 0.58, + 0.6, + 0.62, + 0.64, + 0.66, + 0.68, + 0.7, + 0.72, + 0.74, + 0.76, + 0.78, + 0.8, + 0.82 + ] + ] + ]; +}; diff --git a/frontend/packages/mock/data/roc-curve/steps.ts b/frontend/packages/mock/data/roc-curve/steps.ts new file mode 100644 index 000000000..2c324d6bd --- /dev/null +++ b/frontend/packages/mock/data/roc-curve/steps.ts @@ -0,0 +1,46 @@ +/** + * Copyright 2020 Baidu Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {Request} from 'express'; + +export default (request: Request) => { + if (request.query.run === 'train') { + return [ + [1593069993786.464, 0], + [1593069993787.353, 1], + [1593069993788.1448, 2], + [1593069993788.836, 3], + [1593069993789.4, 4], + [1593069993790.076, 5], + [1593069993790.763, 6], + [1593069993791.473, 7], + [1593069993792.149, 8], + [1593069993792.763, 9] + ]; + } + return [ + [1593069993538.6739, 0], + [1593069993539.396, 1], + [1593069993540.066, 2], + [1593069993540.662, 3], + [1593069993541.333, 4], + [1593069993542.078, 5], + [1593069993543.1821, 6], + [1593069993543.998, 7], + [1593069993544.9128, 8], + [1593069993545.62, 9] + ]; +}; diff --git a/frontend/packages/mock/data/roc-curve/tag.ts b/frontend/packages/mock/data/roc-curve/tag.ts new file mode 100644 index 000000000..e79e9fe25 --- /dev/null +++ b/frontend/packages/mock/data/roc-curve/tag.ts @@ -0,0 +1,29 @@ +/** + * Copyright 2020 Baidu Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default { + runs: ['train', 'test'], + tags: [ + ['layer2/biases/summaries/mean', 'test/1234', 'another'], + [ + 'layer2/biases/summaries/mean', + 'layer2/biases/summaries/accuracy', + 'layer2/biases/summaries/cost', + 'test/431', + 'others' + ] + ] +}; diff --git a/visualdl/component/__init__.py b/visualdl/component/__init__.py index 192c6b49b..4b3ae24ad 100644 --- a/visualdl/component/__init__.py +++ b/visualdl/component/__init__.py @@ -34,6 +34,9 @@ "pr_curve": { "enabled": False }, + "roc_curve": { + "enabled": False + }, "meta_data": { "enabled": False } diff --git a/visualdl/component/base_component.py b/visualdl/component/base_component.py index 7cbfec222..482e7dbeb 100644 --- a/visualdl/component/base_component.py +++ b/visualdl/component/base_component.py @@ -389,3 +389,125 @@ def pr_curve_raw(tag, tp, fp, tn, fn, precision, recall, step, walltime): Record.Value( id=step, tag=tag, timestamp=walltime, pr_curve=prcurve) ]) +def compute_roc_curve(labels, predictions, num_thresholds=None, weights=None): + """ Compute ROC curve data by labels and predictions. + Args: + labels (numpy.ndarray or list): Binary labels for each element. + predictions (numpy.ndarray or list): The probability that an element be + classified as true. + num_thresholds (int): Number of thresholds used to draw the curve. + weights (float): Multiple of data to display on the curve. + """ + if isinstance(labels, list): + labels = np.array(labels) + if isinstance(predictions, list): + predictions = np.array(predictions) + _MINIMUM_COUNT = 1e-7 + + if weights is None: + weights = 1.0 + + bucket_indices = np.int32(np.floor(predictions * (num_thresholds - 1))) + float_labels = labels.astype(np.float) + histogram_range = (0, num_thresholds - 1) + tp_buckets, _ = np.histogram( + bucket_indices, + bins=num_thresholds, + range=histogram_range, + weights=float_labels * weights) + fp_buckets, _ = np.histogram( + bucket_indices, + bins=num_thresholds, + range=histogram_range, + weights=(1.0 - float_labels) * weights) + + # Obtain the reverse cumulative sum. + tp = np.cumsum(tp_buckets[::-1])[::-1] + fp = np.cumsum(fp_buckets[::-1])[::-1] + tn = fp[0] - fp + fn = tp[0] - tp + tpr = tp / np.maximum(_MINIMUM_COUNT, tn + fp) + fpr = fp / np.maximum(_MINIMUM_COUNT, tn + fp) + data = { + 'tp': tp.astype(int).tolist(), + 'fp': fp.astype(int).tolist(), + 'tn': tn.astype(int).tolist(), + 'fn': fn.astype(int).tolist(), + 'tpr': tpr.astype(float).tolist(), + 'fpr': fpr.astype(float).tolist() + } + return data + + +def roc_curve(tag, labels, predictions, step, walltime, num_thresholds=127, + weights=None): + """Package data to one roc_curve. + Args: + tag (string): Data identifier + labels (numpy.ndarray or list): Binary labels for each element. + predictions (numpy.ndarray or list): The probability that an element be + classified as true. + step (int): Step of pr_curve + walltime (int): Wall time of pr_curve + num_thresholds (int): Number of thresholds used to draw the curve. + weights (float): Multiple of data to display on the curve. + Return: + Package with format of record_pb2.Record + """ + num_thresholds = min(num_thresholds, 127) + roc_curve_map = compute_roc_curve(labels, predictions, num_thresholds, weights) + + return roc_curve_raw(tag=tag, + tp=roc_curve_map['tp'], + fp=roc_curve_map['fp'], + tn=roc_curve_map['tn'], + fn=roc_curve_map['fn'], + tpr=roc_curve_map['tpr'], + fpr=roc_curve_map['fpr'], + step=step, + walltime=walltime) + + +def roc_curve_raw(tag, tp, fp, tn, fn, tpr, fpr, step, walltime): + """Package raw data to one roc_curve. + Args: + tag (string): Data identifier + tp (list): True Positive. + fp (list): False Positive. + tn (list): True Negative. + fn (list): False Negative. + tpr (list): true positive rate: + fpr (list): false positive rate. + step (int): Step of roc_curve + walltime (int): Wall time of roc_curve + num_thresholds (int): Number of thresholds used to draw the curve. + weights (float): Multiple of data to display on the curve. + Return: + Package with format of record_pb2.Record + """ + + """ + if isinstance(tp, np.ndarray): + tp = tp.astype(int).tolist() + if isinstance(fp, np.ndarray): + fp = fp.astype(int).tolist() + if isinstance(tn, np.ndarray): + tn = tn.astype(int).tolist() + if isinstance(fn, np.ndarray): + fn = fn.astype(int).tolist() + if isinstance(tpr, np.ndarray): + tpr = tpr.astype(int).tolist() + if isinstance(fpr, np.ndarray): + fpr = fpr.astype(int).tolist() + """ + roc_curve = Record.ROC_Curve(TP=tp, + FP=fp, + TN=tn, + FN=fn, + tpr=tpr, + fpr=fpr) + return Record(values=[ + Record.Value( + id=step, tag=tag, timestamp=walltime, roc_curve=roc_curve) + ]) + diff --git a/visualdl/proto/record.proto b/visualdl/proto/record.proto index 434ca7526..d827cd208 100644 --- a/visualdl/proto/record.proto +++ b/visualdl/proto/record.proto @@ -43,6 +43,15 @@ message Record { repeated double precision = 5; repeated double recall = 6; } + + message ROC_Curve { + repeated int64 TP = 1 [packed = true]; + repeated int64 FP = 2 [packed = true]; + repeated int64 TN = 3 [packed = true]; + repeated int64 FN = 4 [packed = true]; + repeated double tpr = 5; + repeated double fpr = 6; + } message MetaData { string display_name = 1; @@ -60,6 +69,7 @@ message Record { Histogram histogram = 8; PRCurve pr_curve = 9; MetaData meta_data = 10; + ROC_Curve roc_curve = 11; } } diff --git a/visualdl/proto/record_pb2.py b/visualdl/proto/record_pb2.py index 8ce7e333b..883dca51a 100644 --- a/visualdl/proto/record_pb2.py +++ b/visualdl/proto/record_pb2.py @@ -18,7 +18,7 @@ package='visualdl', syntax='proto3', serialized_options=None, - serialized_pb=b'\n\x0crecord.proto\x12\x08visualdl\"\xc8\x07\n\x06Record\x12&\n\x06values\x18\x01 \x03(\x0b\x32\x16.visualdl.Record.Value\x1a%\n\x05Image\x12\x1c\n\x14\x65ncoded_image_string\x18\x04 \x01(\x0c\x1a}\n\x05\x41udio\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x14\n\x0cnum_channels\x18\x02 \x01(\x03\x12\x15\n\rlength_frames\x18\x03 \x01(\x03\x12\x1c\n\x14\x65ncoded_audio_string\x18\x04 \x01(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x05 \x01(\t\x1a+\n\tEmbedding\x12\r\n\x05label\x18\x01 \x03(\t\x12\x0f\n\x07vectors\x18\x02 \x03(\x02\x1aP\n\nEmbeddings\x12.\n\nembeddings\x18\x01 \x03(\x0b\x32\x1a.visualdl.Record.Embedding\x12\x12\n\nlabel_meta\x18\x02 \x03(\t\x1a\x43\n\x10\x62ytes_embeddings\x12\x16\n\x0e\x65ncoded_labels\x18\x01 \x01(\x0c\x12\x17\n\x0f\x65ncoded_vectors\x18\x02 \x01(\x0c\x1a\x34\n\tHistogram\x12\x10\n\x04hist\x18\x01 \x03(\x01\x42\x02\x10\x01\x12\x15\n\tbin_edges\x18\x02 \x03(\x01\x42\x02\x10\x01\x1al\n\x07PRCurve\x12\x0e\n\x02TP\x18\x01 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46P\x18\x02 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02TN\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46N\x18\x04 \x03(\x03\x42\x02\x10\x01\x12\x11\n\tprecision\x18\x05 \x03(\x01\x12\x0e\n\x06recall\x18\x06 \x03(\x01\x1a \n\x08MetaData\x12\x14\n\x0c\x64isplay_name\x18\x01 \x01(\t\x1a\xe5\x02\n\x05Value\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0b\n\x03tag\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\x03\x12\x0f\n\x05value\x18\x04 \x01(\x02H\x00\x12\'\n\x05image\x18\x05 \x01(\x0b\x32\x16.visualdl.Record.ImageH\x00\x12\'\n\x05\x61udio\x18\x06 \x01(\x0b\x32\x16.visualdl.Record.AudioH\x00\x12\x31\n\nembeddings\x18\x07 \x01(\x0b\x32\x1b.visualdl.Record.EmbeddingsH\x00\x12/\n\thistogram\x18\x08 \x01(\x0b\x32\x1a.visualdl.Record.HistogramH\x00\x12,\n\x08pr_curve\x18\t \x01(\x0b\x32\x18.visualdl.Record.PRCurveH\x00\x12.\n\tmeta_data\x18\n \x01(\x0b\x32\x19.visualdl.Record.MetaDataH\x00\x42\x0b\n\tone_valueb\x06proto3' + serialized_pb=b'\n\x0crecord.proto\x12\x08visualdl\"\xcc\x08\n\x06Record\x12&\n\x06values\x18\x01 \x03(\x0b\x32\x16.visualdl.Record.Value\x1a%\n\x05Image\x12\x1c\n\x14\x65ncoded_image_string\x18\x04 \x01(\x0c\x1a}\n\x05\x41udio\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x14\n\x0cnum_channels\x18\x02 \x01(\x03\x12\x15\n\rlength_frames\x18\x03 \x01(\x03\x12\x1c\n\x14\x65ncoded_audio_string\x18\x04 \x01(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x05 \x01(\t\x1a+\n\tEmbedding\x12\r\n\x05label\x18\x01 \x01(\t\x12\x0f\n\x07vectors\x18\x02 \x03(\x02\x1a<\n\nEmbeddings\x12.\n\nembeddings\x18\x01 \x03(\x0b\x32\x1a.visualdl.Record.Embedding\x1a\x43\n\x10\x62ytes_embeddings\x12\x16\n\x0e\x65ncoded_labels\x18\x01 \x01(\x0c\x12\x17\n\x0f\x65ncoded_vectors\x18\x02 \x01(\x0c\x1a\x34\n\tHistogram\x12\x10\n\x04hist\x18\x01 \x03(\x01\x42\x02\x10\x01\x12\x15\n\tbin_edges\x18\x02 \x03(\x01\x42\x02\x10\x01\x1al\n\x07PRCurve\x12\x0e\n\x02TP\x18\x01 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46P\x18\x02 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02TN\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46N\x18\x04 \x03(\x03\x42\x02\x10\x01\x12\x11\n\tprecision\x18\x05 \x03(\x01\x12\x0e\n\x06recall\x18\x06 \x03(\x01\x1a\x65\n\tROC_Curve\x12\x0e\n\x02TP\x18\x01 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46P\x18\x02 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02TN\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46N\x18\x04 \x03(\x03\x42\x02\x10\x01\x12\x0b\n\x03tpr\x18\x05 \x03(\x01\x12\x0b\n\x03\x66pr\x18\x06 \x03(\x01\x1a \n\x08MetaData\x12\x14\n\x0c\x64isplay_name\x18\x01 \x01(\t\x1a\x96\x03\n\x05Value\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0b\n\x03tag\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\x03\x12\x0f\n\x05value\x18\x04 \x01(\x02H\x00\x12\'\n\x05image\x18\x05 \x01(\x0b\x32\x16.visualdl.Record.ImageH\x00\x12\'\n\x05\x61udio\x18\x06 \x01(\x0b\x32\x16.visualdl.Record.AudioH\x00\x12\x31\n\nembeddings\x18\x07 \x01(\x0b\x32\x1b.visualdl.Record.EmbeddingsH\x00\x12/\n\thistogram\x18\x08 \x01(\x0b\x32\x1a.visualdl.Record.HistogramH\x00\x12,\n\x08pr_curve\x18\t \x01(\x0b\x32\x18.visualdl.Record.PRCurveH\x00\x12.\n\tmeta_data\x18\n \x01(\x0b\x32\x19.visualdl.Record.MetaDataH\x00\x12/\n\troc_curve\x18\x0b \x01(\x0b\x32\x1a.visualdl.Record.ROC_CurveH\x00\x42\x0b\n\tone_valueb\x06proto3' ) @@ -325,6 +325,71 @@ serialized_end=601, ) +_RECORD_ROC_CURVE = _descriptor.Descriptor( + name='ROC_Curve', + full_name='visualdl.Record.ROC_Curve', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='TP', full_name='visualdl.Record.ROC_Curve.TP', index=0, + number=1, type=3, cpp_type=2, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\020\001', file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='FP', full_name='visualdl.Record.ROC_Curve.FP', index=1, + number=2, type=3, cpp_type=2, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\020\001', file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='TN', full_name='visualdl.Record.ROC_Curve.TN', index=2, + number=3, type=3, cpp_type=2, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\020\001', file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='FN', full_name='visualdl.Record.ROC_Curve.FN', index=3, + number=4, type=3, cpp_type=2, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\020\001', file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tpr', full_name='visualdl.Record.ROC_Curve.tpr', index=4, + number=5, type=1, cpp_type=5, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='fpr', full_name='visualdl.Record.ROC_Curve.fpr', index=5, + number=6, type=1, cpp_type=5, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=583, + serialized_end=684, +) + _RECORD_METADATA = _descriptor.Descriptor( name='MetaData', full_name='visualdl.Record.MetaData', @@ -351,8 +416,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=603, - serialized_end=635, + serialized_start=686, + serialized_end=718, ) _RECORD_VALUE = _descriptor.Descriptor( @@ -432,6 +497,13 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='roc_curve', full_name='visualdl.Record.Value.roc_curve', index=10, + number=11, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -447,8 +519,8 @@ name='one_value', full_name='visualdl.Record.Value.one_value', index=0, containing_type=None, fields=[]), ], - serialized_start=638, - serialized_end=995, + serialized_start=721, + serialized_end=1127, ) _RECORD = _descriptor.Descriptor( @@ -468,7 +540,7 @@ ], extensions=[ ], - nested_types=[_RECORD_IMAGE, _RECORD_AUDIO, _RECORD_EMBEDDING, _RECORD_EMBEDDINGS, _RECORD_BYTES_EMBEDDINGS, _RECORD_HISTOGRAM, _RECORD_PRCURVE, _RECORD_METADATA, _RECORD_VALUE, ], + nested_types=[_RECORD_IMAGE, _RECORD_AUDIO, _RECORD_EMBEDDING, _RECORD_EMBEDDINGS, _RECORD_BYTES_EMBEDDINGS, _RECORD_HISTOGRAM, _RECORD_PRCURVE, _RECORD_ROC_CURVE, _RECORD_METADATA, _RECORD_VALUE, ], enum_types=[ ], serialized_options=None, @@ -478,7 +550,7 @@ oneofs=[ ], serialized_start=27, - serialized_end=995, + serialized_end=1127, ) _RECORD_IMAGE.containing_type = _RECORD @@ -489,6 +561,7 @@ _RECORD_BYTES_EMBEDDINGS.containing_type = _RECORD _RECORD_HISTOGRAM.containing_type = _RECORD _RECORD_PRCURVE.containing_type = _RECORD +_RECORD_ROC_CURVE.containing_type = _RECORD _RECORD_METADATA.containing_type = _RECORD _RECORD_VALUE.fields_by_name['image'].message_type = _RECORD_IMAGE _RECORD_VALUE.fields_by_name['audio'].message_type = _RECORD_AUDIO @@ -496,6 +569,7 @@ _RECORD_VALUE.fields_by_name['histogram'].message_type = _RECORD_HISTOGRAM _RECORD_VALUE.fields_by_name['pr_curve'].message_type = _RECORD_PRCURVE _RECORD_VALUE.fields_by_name['meta_data'].message_type = _RECORD_METADATA +_RECORD_VALUE.fields_by_name['roc_curve'].message_type = _RECORD_ROC_CURVE _RECORD_VALUE.containing_type = _RECORD _RECORD_VALUE.oneofs_by_name['one_value'].fields.append( _RECORD_VALUE.fields_by_name['value']) @@ -518,6 +592,9 @@ _RECORD_VALUE.oneofs_by_name['one_value'].fields.append( _RECORD_VALUE.fields_by_name['meta_data']) _RECORD_VALUE.fields_by_name['meta_data'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value'] +_RECORD_VALUE.oneofs_by_name['one_value'].fields.append( + _RECORD_VALUE.fields_by_name['roc_curve']) +_RECORD_VALUE.fields_by_name['roc_curve'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value'] _RECORD.fields_by_name['values'].message_type = _RECORD_VALUE DESCRIPTOR.message_types_by_name['Record'] = _RECORD _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -573,6 +650,13 @@ }) , + 'ROC_Curve' : _reflection.GeneratedProtocolMessageType('ROC_Curve', (_message.Message,), { + 'DESCRIPTOR' : _RECORD_ROC_CURVE, + '__module__' : 'record_pb2' + # @@protoc_insertion_point(class_scope:visualdl.Record.ROC_Curve) + }) + , + 'MetaData' : _reflection.GeneratedProtocolMessageType('MetaData', (_message.Message,), { 'DESCRIPTOR' : _RECORD_METADATA, '__module__' : 'record_pb2' @@ -598,6 +682,7 @@ _sym_db.RegisterMessage(Record.bytes_embeddings) _sym_db.RegisterMessage(Record.Histogram) _sym_db.RegisterMessage(Record.PRCurve) +_sym_db.RegisterMessage(Record.ROC_Curve) _sym_db.RegisterMessage(Record.MetaData) _sym_db.RegisterMessage(Record.Value) @@ -608,4 +693,8 @@ _RECORD_PRCURVE.fields_by_name['FP']._options = None _RECORD_PRCURVE.fields_by_name['TN']._options = None _RECORD_PRCURVE.fields_by_name['FN']._options = None +_RECORD_ROC_CURVE.fields_by_name['TP']._options = None +_RECORD_ROC_CURVE.fields_by_name['FP']._options = None +_RECORD_ROC_CURVE.fields_by_name['TN']._options = None +_RECORD_ROC_CURVE.fields_by_name['FN']._options = None # @@protoc_insertion_point(module_scope) diff --git a/visualdl/reader/reader.py b/visualdl/reader/reader.py index 63423e8a3..6cf39902e 100644 --- a/visualdl/reader/reader.py +++ b/visualdl/reader/reader.py @@ -150,6 +150,8 @@ def parse_from_bin(self, record_bin): component = "histogram" elif "pr_curve" == value_type: component = "pr_curve" + elif "roc_curve" == value_type: + component = "roc_curve" elif "meta_data" == value_type: self.update_meta_data(record) component = "meta_data" diff --git a/visualdl/server/api.py b/visualdl/server/api.py index 1026142fb..4db5264e1 100644 --- a/visualdl/server/api.py +++ b/visualdl/server/api.py @@ -112,6 +112,9 @@ def embedding_tags(self): @result() def pr_curve_tags(self): return self._get_with_retry('data/plugin/pr_curves/tags', lib.get_pr_curve_tags) + @result() + def roc_curve_tags(self): + return self._get_with_retry('data/plugin/roc_curves/tags', lib.get_roc_curve_tags) @result() def scalar_list(self, run, tag): @@ -173,11 +176,18 @@ def histogram_list(self, run, tag): def pr_curves_pr_curve(self, run, tag): key = os.path.join('data/plugin/pr_curves/pr_curve', run, tag) return self._get_with_retry(key, lib.get_pr_curve, run, tag) - + @result() + def roc_curves_roc_curve(self, run, tag): + key = os.path.join('data/plugin/roc_curves/roc_curve', run, tag) + return self._get_with_retry(key, lib.get_roc_curve, run, tag) @result() def pr_curves_steps(self, run): key = os.path.join('data/plugin/pr_curves/steps', run) return self._get_with_retry(key, lib.get_pr_curve_step, run) + @result() + def roc_curves_steps(self, run): + key = os.path.join('data/plugin/roc_curves/steps', run) + return self._get_with_retry(key, lib.get_roc_curve_step, run) @result('application/octet-stream', lambda s: {"Content-Disposition": 'attachment; filename="%s"' % s.model_name} if len(s.model_name) else None) def graph_graph(self): @@ -198,6 +208,7 @@ def create_api_call(logdir, model, cache_timeout): 'embedding/tags': (api.embedding_tags, []), 'histogram/tags': (api.histogram_tags, []), 'pr-curve/tags': (api.pr_curve_tags, []), + 'roc-curve/tags': (api.roc_curve_tags, []), 'scalar/list': (api.scalar_list, ['run', 'tag']), 'image/list': (api.image_list, ['run', 'tag']), 'image/image': (api.image_image, ['run', 'tag', 'index']), @@ -210,7 +221,9 @@ def create_api_call(logdir, model, cache_timeout): 'histogram/list': (api.histogram_list, ['run', 'tag']), 'graph/graph': (api.graph_graph, []), 'pr-curve/list': (api.pr_curves_pr_curve, ['run', 'tag']), - 'pr-curve/steps': (api.pr_curves_steps, ['run']) + 'roc-curve/list': (api.roc_curves_roc_curve, ['run', 'tag']), + 'pr-curve/steps': (api.pr_curves_steps, ['run']), + 'roc-curve/steps': (api.roc_curves_steps, ['run']) } def call(path: str, args): diff --git a/visualdl/server/data_manager.py b/visualdl/server/data_manager.py index 7b74d4cec..9acadcdf3 100644 --- a/visualdl/server/data_manager.py +++ b/visualdl/server/data_manager.py @@ -24,6 +24,7 @@ "embeddings": 50000000, "audio": 10, "pr_curve": 300, + "roc_curve": 300, "meta_data": 100 } @@ -346,6 +347,8 @@ def __init__(self): Reservoir(max_size=DEFAULT_PLUGIN_MAXSIZE["audio"]), "pr_curve": Reservoir(max_size=DEFAULT_PLUGIN_MAXSIZE["pr_curve"]), + "roc_curve": + Reservoir(max_size=DEFAULT_PLUGIN_MAXSIZE["roc_curve"]), "meta_data": Reservoir(max_size=DEFAULT_PLUGIN_MAXSIZE["meta_data"]) } diff --git a/visualdl/server/lib.py b/visualdl/server/lib.py index b0c7b452a..4b9316014 100644 --- a/visualdl/server/lib.py +++ b/visualdl/server/lib.py @@ -186,6 +186,28 @@ def get_pr_curve(log_reader, run, tag): list(pr_curve.FN), num_thresholds]) return results + + +def get_roc_curve(log_reader, run, tag): + run = log_reader.name2tags[run] if run in log_reader.name2tags else run + log_reader.load_new_data() + records = log_reader.data_manager.get_reservoir("roc_curve").get_items( + run, decode_tag(tag)) + results = [] + for item in records: + roc_curve = item.roc_curve + length = len(roc_curve.tpr) + num_thresholds = [float(v) / length for v in range(1, length + 1)] + results.append([s2ms(item.timestamp), + item.id, + list(roc_curve.tpr), + list(roc_curve.fpr), + list(roc_curve.TP), + list(roc_curve.FP), + list(roc_curve.TN), + list(roc_curve.FN), + num_thresholds]) + return results def get_pr_curve_step(log_reader, run, tag=None): @@ -200,6 +222,18 @@ def get_pr_curve_step(log_reader, run, tag=None): return results +def get_roc_curve_step(log_reader, run, tag=None): + fake_run = run + run = log_reader.name2tags[run] if run in log_reader.name2tags else run + run2tag = get_roc_curve_tags(log_reader) + tag = run2tag['tags'][run2tag['runs'].index(fake_run)][0] + log_reader.load_new_data() + records = log_reader.data_manager.get_reservoir("roc_curve").get_items( + run, decode_tag(tag)) + results = [[s2ms(item.timestamp), item.id] for item in records] + return results + + def get_embeddings_list(log_reader): run2tag = get_logs(log_reader, 'embeddings') @@ -254,30 +288,6 @@ def get_embedding_tensors(log_reader, name): return vectors -def get_embeddings(log_reader, run, tag, reduction, dimension=2): - run = log_reader.name2tags[run] if run in log_reader.name2tags else run - log_reader.load_new_data() - records = log_reader.data_manager.get_reservoir("embeddings").get_items( - run, decode_tag(tag)) - - labels = [] - vectors = [] - for item in records[0].embeddings.embeddings: - labels.append(item.label) - vectors.append(item.vectors) - vectors = np.array(vectors) - - if reduction == 'tsne': - import visualdl.server.tsne as tsne - low_dim_embs = tsne.tsne( - vectors, dimension, initial_dims=50, perplexity=30.0) - - elif reduction == 'pca': - low_dim_embs = simple_pca(vectors, dimension) - - return {"embedding": low_dim_embs.tolist(), "labels": labels} - - def get_histogram(log_reader, run, tag): run = log_reader.name2tags[run] if run in log_reader.name2tags else run log_reader.load_new_data() diff --git a/visualdl/writer/writer.py b/visualdl/writer/writer.py index 9aceabebf..df5f1997f 100644 --- a/visualdl/writer/writer.py +++ b/visualdl/writer/writer.py @@ -16,7 +16,7 @@ import time import numpy as np from visualdl.writer.record_writer import RecordFileWriter -from visualdl.component.base_component import scalar, image, embedding, audio, histogram, pr_curve, meta_data +from visualdl.component.base_component import scalar, image, embedding, audio, histogram, pr_curve, roc_curve, meta_data class DummyFileWriter(object): @@ -380,6 +380,48 @@ def add_pr_curve(self, num_thresholds=num_thresholds, weights=weights )) + def add_roc_curve(self, + tag, + labels, + predictions, + step, + num_thresholds=10, + weights=None, + walltime=None): + """Add an ROC curve to vdl record file. + Args: + tag (string): Data identifier + labels (numpy.ndarray or list): Binary labels for each element. + predictions (numpy.ndarray or list): The probability that an element + be classified as true. + step (int): Step of pr curve. + weights (float): Multiple of data to display on the curve. + num_thresholds (int): Number of thresholds used to draw the curve. + walltime (int): Wall time of pr curve. + Example: + with LogWriter(logdir="./log/roc_curve_test/train") as writer: + for index in range(3): + labels = np.random.randint(2, size=100) + predictions = np.random.rand(100) + writer.add_roc_curve(tag='default', + labels=labels, + predictions=predictions, + step=index) + """ + if '%' in tag: + raise RuntimeError("% can't appear in tag!") + walltime = round(time.time() * 1000) if walltime is None else walltime + self._get_file_writer().add_record( + roc_curve( + tag=tag, + labels=labels, + predictions=predictions, + step=step, + walltime=walltime, + num_thresholds=num_thresholds, + weights=weights + )) + def flush(self): """Flush all data in cache to disk.