Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

rewrite roc curve #881

Merged
merged 18 commits into from
Dec 23, 2020
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge branch 'develop' into develop
  • Loading branch information
PeterPanZH authored Dec 22, 2020
commit 8402fdf2de50e66f615181ac907784d694639d9f
58 changes: 44 additions & 14 deletions visualdl/server/lib.py
Original file line number Diff line number Diff line change
@@ -233,29 +233,59 @@ def get_roc_curve_step(log_reader, run, tag=None):
results = [[s2ms(item.timestamp), item.id] for item in records]
return results


def get_embeddings(log_reader, run, tag, reduction, dimension=2):
run = log_reader.name2tags[run] if run in log_reader.name2tags else run

def get_embeddings_list(log_reader):
run2tag = get_logs(log_reader, 'embeddings')

for run, _tags in zip(run2tag['runs'], run2tag['tags']):
for tag in _tags:
name = path = os.path.join(run, tag)
if name in EMBEDDING_NAME:
return embedding_names
EMBEDDING_NAME.update({name: {'run': run, 'tag': tag}})
records = log_reader.data_manager.get_reservoir("embeddings").get_items(
run, decode_tag(tag))
row_len = len(records[0].embeddings.embeddings)
col_len = len(records[0].embeddings.embeddings[0].vectors)
shape = [row_len, col_len]
embedding_names.append({'name': name, 'shape': shape, 'path': path})
return embedding_names


def get_embedding_labels(log_reader, name):
run = EMBEDDING_NAME[name]['run']
tag = EMBEDDING_NAME[name]['tag']
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("embeddings").get_items(
run, decode_tag(tag))

labels = []
vectors = []
for item in records[0].embeddings.embeddings:
labels.append(item.label)
vectors.append(item.vectors)
vectors = np.array(vectors)

if reduction == 'tsne':
import visualdl.server.tsne as tsne
low_dim_embs = tsne.tsne(
vectors, dimension, initial_dims=50, perplexity=30.0)
label_meta = records[0].embeddings.label_meta
if label_meta:
labels = [label_meta] + labels

with io.StringIO() as fp:
csv_writer = csv.writer(fp, delimiter='\t')
csv_writer.writerows(labels)
labels = fp.getvalue()

# labels = "\n".join(str(i) for i in labels)
return labels

elif reduction == 'pca':
low_dim_embs = simple_pca(vectors, dimension)

return {"embedding": low_dim_embs.tolist(), "labels": labels}
def get_embedding_tensors(log_reader, name):
run = EMBEDDING_NAME[name]['run']
tag = EMBEDDING_NAME[name]['tag']
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("embeddings").get_items(
run, decode_tag(tag))
vectors = []
for item in records[0].embeddings.embeddings:
vectors.append(item.vectors)
vectors = np.array(vectors).flatten().astype(np.float32).tobytes()
return vectors


def get_histogram(log_reader, run, tag):
You are viewing a condensed version of this merge commit. You can view the full changes here.