Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(translations/differences): visualize how many commits behind translations are (#8338) #8338

Merged
merged 23 commits into from
Apr 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
fa4bea8
PoC: visualization for how far behind the lastest commit
hochan222 Mar 4, 2023
6541ebc
feat: tag for Metadata does not exist.
hochan222 Mar 4, 2023
19b5733
fix: filter available
hochan222 Mar 6, 2023
6c9af9b
refactor: execSync cwd option
hochan222 Mar 6, 2023
1f50870
fix: filter priority
hochan222 Mar 6, 2023
36e4628
Optimization through cache
hochan222 Mar 6, 2023
9ebd1a6
reduce ja time to 214s with git rev-list
hochan222 Mar 11, 2023
9d3d8f7
feat: source commit error report
hochan222 Mar 11, 2023
b04917e
Optimize source commit logic
hochan222 Mar 11, 2023
528a7e4
Merge branch 'main' into visualization-source-commit
hochan222 Mar 11, 2023
06e74e3
feat: source commit importance color
hochan222 Mar 11, 2023
ad5246d
new approach for traversing git commit graph
hochan222 Mar 25, 2023
a71eae9
remove p tag in table tag
hochan222 Mar 25, 2023
153d681
add details to report message and file name
hochan222 Mar 25, 2023
94eee57
modify source commit filter condition
hochan222 Mar 25, 2023
2d2dba7
fix: consider sourceCommitsBehindCount undefined
hochan222 Mar 25, 2023
032db13
check automatic cache validation
hochan222 Mar 26, 2023
c799cec
refactor: types and naming
hochan222 Mar 26, 2023
08701ad
execSync -> spawn, various fixes for optimization
hochan222 Apr 22, 2023
afcee51
fix: make sourceCommitCache work correctly
hochan222 Apr 22, 2023
cf36eb5
remove log
hochan222 Apr 22, 2023
0e84df6
Merge branch 'main' into visualization-source-commit
hochan222 Mar 25, 2024
9997b2c
Merge branch 'main' into visualization-source-commit
caugner Apr 4, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ testing/content/files/en-us/markdown/tool/m2h/index.html
# eslintcache
client/.eslintcache
popularities.json
source-commit.json
source-commit-invalid-report.txt

client/src/.linaria_cache
ssr/.linaria-cache/
Expand Down
38 changes: 38 additions & 0 deletions client/src/translations/differences/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ interface DocumentEdits {
parentModified: string;
commitURL: string;
parentCommitURL: string;
sourceCommitsBehindCount?: number;
sourceCommitURL?: string;
}

interface Document {
Expand Down Expand Up @@ -574,6 +576,10 @@ function DocumentsTable({
const a = A.mdn_url;
const b = B.mdn_url;
return reverse * a.localeCompare(b);
} else if (sort === "sourceCommit") {
const a = A.edits.sourceCommitsBehindCount ?? -1;
const b = B.edits.sourceCommitsBehindCount ?? -1;
return reverse * (b - a);
} else {
throw new Error(`Unrecognized sort '${sort}'`);
}
Expand Down Expand Up @@ -606,6 +612,7 @@ function DocumentsTable({
<TableHead id="popularity" title="Popularity" />
<TableHead id="modified" title="Last modified" />
<TableHead id="differences" title="Differences" />
<TableHead id="sourceCommit" title="Source Commit" />
</tr>
</thead>
<tbody>
Expand Down Expand Up @@ -658,6 +665,14 @@ function DocumentsTable({
<LastModified edits={doc.edits} />
</td>
<td>{doc.differences.total.toLocaleString()}</td>
<td>
<L10nSourceCommitModified
sourceCommitsBehindCount={
doc.edits.sourceCommitsBehindCount
}
sourceCommitURL={doc.edits.sourceCommitURL}
/>
</td>
</tr>
);
})}
Expand All @@ -680,6 +695,29 @@ function DocumentsTable({
);
}

function L10nSourceCommitModified({
sourceCommitsBehindCount,
sourceCommitURL,
}: Pick<DocumentEdits, "sourceCommitsBehindCount" | "sourceCommitURL">) {
if (
!sourceCommitURL ||
(!sourceCommitsBehindCount && sourceCommitsBehindCount !== 0)
) {
return <>Metadata does not exist.</>;
}

const getImportanceColor = () => {
if (sourceCommitsBehindCount === 0) return "🟢";
return sourceCommitsBehindCount < 10 ? "🟠" : "🔴";
};

return (
<a
href={sourceCommitURL}
>{`${getImportanceColor()} ${sourceCommitsBehindCount} commits behind`}</a>
);
}

function LastModified({ edits }: { edits: DocumentEdits }) {
const modified = dayjs(edits.modified);
const parentModified = dayjs(edits.parentModified);
Expand Down
212 changes: 196 additions & 16 deletions server/translations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import path from "node:path";

import express from "express";
import { fdir } from "fdir";
import { execSync, spawn } from "node:child_process";

import { getPopularities, Document, Translation } from "../content/index.js";
import {
Expand Down Expand Up @@ -59,8 +60,52 @@ function packageTranslationDifferences(translationDifferences) {
return { total, countByType };
}

type RecentRepoHashType = string;

const _foundDocumentsCache = new Map();
const sourceCommitCache = fs.existsSync("./source-commit.json")
? new Map<string, number | RecentRepoHashType>(
Object.entries(
JSON.parse(fs.readFileSync("./source-commit.json", "utf8"))
)
)
: new Map<string, number>();
const commitFiles = new Map<string, string[]>();
let commitFilesOldest = "HEAD";
export async function findDocuments({ locale }) {
function checkCacheValidation(prevCache: Map<any, any>): void {
const contentHash = getRecentRepoHash(CONTENT_ROOT);
const translatedContentHash = getRecentRepoHash(CONTENT_TRANSLATED_ROOT);

function getRecentRepoHash(cwd: string): string {
return execSync("git rev-parse HEAD", { cwd }).toString().trimEnd();
}
function updateRecentRepoHash(cache: Map<string, any>): void {
cache.set(CONTENT_ROOT, contentHash);
cache.set(CONTENT_TRANSLATED_ROOT, translatedContentHash);
}
function isValidCache(cache: Map<string, any>): boolean {
return (
cache.has(CONTENT_ROOT) &&
cache.has(CONTENT_TRANSLATED_ROOT) &&
cache.get(CONTENT_ROOT) === contentHash &&
cache.get(CONTENT_TRANSLATED_ROOT) === translatedContentHash
);
}

if (isValidCache(sourceCommitCache)) {
return;
}
if (!isValidCache(prevCache)) {
prevCache.clear();
sourceCommitCache.clear();
hochan222 marked this conversation as resolved.
Show resolved Hide resolved
commitFiles.clear();
commitFilesOldest = "HEAD";
updateRecentRepoHash(prevCache);
updateRecentRepoHash(sourceCommitCache);
}
}

const counts = {
// Number of documents found that aren't skipped
found: 0,
Expand All @@ -81,6 +126,7 @@ export async function findDocuments({ locale }) {
});
counts.total = documentsFound.count;

checkCacheValidation(_foundDocumentsCache);
hochan222 marked this conversation as resolved.
Show resolved Hide resolved
if (!_foundDocumentsCache.has(locale)) {
_foundDocumentsCache.set(locale, new Map());
}
Expand All @@ -91,7 +137,7 @@ export async function findDocuments({ locale }) {

if (!cache.has(filePath) || cache.get(filePath).mtime < mtime) {
counts.cacheMisses++;
const document = getDocument(filePath);
const document = await getDocument(filePath);
cache.set(filePath, {
document,
mtime,
Expand All @@ -114,14 +160,20 @@ export async function findDocuments({ locale }) {
took,
};

fs.writeFileSync(
"./source-commit.json",
JSON.stringify(Object.fromEntries(sourceCommitCache)),
"utf8"
);

return {
counts,
times,
documents,
};
}

function getDocument(filePath) {
async function getDocument(filePath) {
function packagePopularity(document, parentDocument) {
return {
value: document.metadata.popularity,
Expand All @@ -141,34 +193,162 @@ function getDocument(filePath) {
};
}

function packageEdits(document, parentDocument) {
const commitURL = getLastCommitURL(
document.fileInfo.root,
document.metadata.hash
);
const parentCommitURL = getLastCommitURL(
parentDocument.fileInfo.root,
parentDocument.metadata.hash
);
const modified = document.metadata.modified;
const parentModified = parentDocument.metadata.modified;
function recordInvalidSourceCommit(
fileFolder: string,
commitHash: string,
message: string
) {
const filePath = "./source-commit-invalid-report.txt";
const errorMessage = `- ${commitHash} commit hash is invalid in ${fileFolder}: ${message.replace(
/\n/g,
" "
)}`;
if (!fs.existsSync(filePath)) {
fs.writeFileSync(filePath, "");
}

fs.appendFile(filePath, `${errorMessage}\n`, function (err) {
if (err) throw err;
});
}

class GitError extends Error {
constructor(stderr: string) {
super(stderr);
this.name = "GitError";
}
}

function fillMemStore(commitHash: string) {
return new Promise((resolve, reject) => {
const git = spawn(
"git",
[
"log",
"--pretty=format:%x00%x00%H",
"--name-only",
"-z",
`${commitHash}..${commitFilesOldest}`,
],
{
cwd: CONTENT_ROOT,
}
);

let stdoutBuffer = "";

git.stdout.on("data", (data) => {
stdoutBuffer += data.toString();
const commits = stdoutBuffer.split("\0\0");
const partial = commits.pop();
stdoutBuffer = partial;
commits.forEach((commit) => {
const [dirtyHash, files] = commit.split("\n");
// necessary for commits following those with no changes:
const hash = dirtyHash.replace(/\0/g, "");
commitFiles.set(hash, files ? files.split("\0") : []);
});
});

let stderr = "";

git.stderr.on("data", (data) => {
stderr += data.toString();
});

git.on("close", (code) => {
commitFilesOldest = commitHash;
code ? reject(new GitError(stderr)) : resolve(null);
});
});
}

async function getCommitBehindFromLatest(
fileFolder: string,
parentFilePath: string,
commitHash: string
): Promise<number> {
if (sourceCommitCache.has(fileFolder)) {
return sourceCommitCache.get(fileFolder) as number;
}

try {
let count = 0;
if (!commitFiles.has(commitHash)) {
await fillMemStore(commitHash);
}
for (const [hash, files] of commitFiles.entries()) {
if (hash === commitHash) {
if (!files.includes(parentFilePath)) {
recordInvalidSourceCommit(
fileFolder,
commitHash,
"file isn't changed in this commit"
);
}
break;
}
if (files.includes(parentFilePath)) count++;
}
sourceCommitCache.set(fileFolder, count);
} catch (err) {
if (err instanceof GitError) {
recordInvalidSourceCommit(fileFolder, commitHash, err.message);
} else {
throw err;
}
}

return sourceCommitCache.get(fileFolder) as number;
}

async function packageEdits(document, parentDocument) {
const {
fileInfo: { root: fileRoot, folder: fileFolder },
metadata: { hash: fileHash, modified, l10n },
} = document;
const {
fileInfo: { root: parentFileRoot, path: parentFilePath },
metadata: { hash: parentFileHash, parentModified },
} = parentDocument;

const commitURL = getLastCommitURL(fileRoot, fileHash);
const parentCommitURL = getLastCommitURL(parentFileRoot, parentFileHash);
let sourceCommitURL;
let sourceCommitsBehindCount;

if (l10n?.sourceCommit) {
sourceCommitURL = getLastCommitURL(CONTENT_ROOT, l10n.sourceCommit);
sourceCommitsBehindCount = await getCommitBehindFromLatest(
fileFolder,
parentFilePath.replace(parentFileRoot, "files"),
l10n.sourceCommit
);
}

return {
commitURL,
parentCommitURL,
modified,
parentModified,
sourceCommitURL,
sourceCommitsBehindCount,
};
}

// We can't just open the `index.json` and return it like that in the XHR
// payload. It's too much stuff and some values need to be repackaged/
// serialized or some other transformation computation.
function packageDocument(document, englishDocument, translationDifferences) {
async function packageDocument(
document,
englishDocument,
translationDifferences
) {
const mdn_url = document.url;
const { title } = document.metadata;
const popularity = packagePopularity(document, englishDocument);
const differences = packageTranslationDifferences(translationDifferences);
const edits = packageEdits(document, englishDocument);
const edits = await packageEdits(document, englishDocument);
return { popularity, differences, edits, mdn_url, title };
}

Expand All @@ -191,7 +371,7 @@ function getDocument(filePath) {
)) {
differences.push(difference);
}
return packageDocument(document, englishDocument, differences);
return await packageDocument(document, englishDocument, differences);
}

const _defaultLocaleDocumentsCache = new Map();
Expand Down
Loading