Skip to content
This repository has been archived by the owner on Jun 23, 2023. It is now read-only.

Commit

Permalink
3rd Party App Repo Support
Browse files Browse the repository at this point in the history

Co-authored-by: Steven Briscoe <me@stevenbriscoe.com>
  • Loading branch information
nevets963 and Steven Briscoe authored Oct 24, 2022
1 parent 45629ed commit eb043b7
Show file tree
Hide file tree
Showing 10 changed files with 257 additions and 48 deletions.
2 changes: 2 additions & 0 deletions app.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ const account = require('routes/v1/account.js');
const system = require('routes/v1/system.js');
const external = require('routes/v1/external.js');
const apps = require('routes/v1/apps.js');
const communityAppStores = require('routes/v1/community-app-stores.js');
const constants = require('utils/const.js');

const app = express();
Expand Down Expand Up @@ -52,6 +53,7 @@ app.use('/v1/account', account);
app.use('/v1/system', system);
app.use('/v1/external', external);
app.use('/v1/apps', apps);
app.use('/v1/community-app-stores', communityAppStores);

app.use(errorHandleMiddleware);
app.use((req, res) => {
Expand Down
101 changes: 68 additions & 33 deletions logic/apps.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ const reposLogic = require('logic/repos.js');
const NodeError = require('models/errors.js').NodeError;
const deriveEntropy = require('modules/derive-entropy');
const constants = require('utils/const.js');
const YAML = require('yaml');
const semver = require('semver');
const path = require('path');

Expand All @@ -21,9 +20,7 @@ async function getAppManifest(folder, appId, manifestFilename) {
let app;
try {
const appYamlPath = path.join(folder, appId, manifestFilename);
const appYaml = await diskService.readFile(appYamlPath, "utf-8");

app = YAML.parse(appYaml);
app = await diskService.readYamlFile(appYamlPath);
} catch(e) {
throw new NodeError(`Failed to parse ${appId} manifest file`);
}
Expand Down Expand Up @@ -55,11 +52,25 @@ async function addAppMetadata(apps) {
app.defaultPassword = '';
}
}));

// Set some app update defaults

apps = apps.map((app) => {
// Set app update defaults
app.updateAvailable = false;

// Set an icon property if it doesn't exist
// Default to using Umbrel gallery assets
if(app.icon === undefined) {
app.icon = `${constants.UMBREL_GALLERY_ASSETS_BASE_URL}/${app.id}/icon.svg`;
}

app.gallery = app.gallery.map(image => {
if(image.startsWith('http://') || image.startsWith('https://')) {
return image;
}

return `${constants.UMBREL_GALLERY_ASSETS_BASE_URL}/${app.id}/${image}`;
});

return app;
});

Expand Down Expand Up @@ -87,39 +98,56 @@ async function get(query) {
return getInstalled(user);
}

// Read all app yaml files within the active app repo
const activeAppRepoFolder = path.join(constants.REPOS_DIR, reposLogic.getId(user));
const repos = await reposLogic.all(user);
const repo = repos.find(repo => {
return repo.id === query.repo;
});

if(repo === undefined) {
throw new NodeError(`Unable to locate repo: ${query.repo}`);
}

let apps = [];

// Read all app yaml files from a given app repo
const activeAppRepoFolder = path.join(constants.REPOS_DIR, reposLogic.slug(repo.url));

// Ignore dot/hidden folders
let appIds = [];
try {
// Ignore dot/hidden folders
appIds = (await diskService.listDirsInDir(activeAppRepoFolder)).filter(folder => folder[0] !== '.');
} catch(e) {
console.error(e);
console.error(`Error reading directory: ${activeAppRepoFolder}`, e);
}

try {
let appsInRepo = await Promise.allSettled(appIds.map(appId => getAppManifest(activeAppRepoFolder, appId, APP_MANIFEST_FILENAME)));

let apps = await Promise.allSettled(appIds.map(appId => getAppManifest(activeAppRepoFolder, appId, APP_MANIFEST_FILENAME)));

apps = filterMapFulfilled(apps);
appsInRepo = filterMapFulfilled(appsInRepo);

// Map some metadata onto each app object
apps = await addAppMetadata(apps);
// Map some metadata onto each app object
apps = await addAppMetadata(appsInRepo);
} catch(e) {
console.error(`Error reading app manifest`, e);
}

let installedAppsMap = {};
user.installedApps.forEach(function(appId){
installedAppsMap[appId] = 1;
})
// If the repo is a community app store
// We need to check if the app id is prefixed (or namespaced) with the repo id
// (defined inside of the umbrel-app-store.yml)
if(repo.id !== constants.UMBREL_APP_STORE_REPO.id) {
apps = apps.filter(app => {
return app.id.startsWith(`${repo.id}-`);
});
}

// Let's now check whether any have an app update
await Promise.all(apps.map(async app => {
// Ignore apps that are not installed
if(installedAppsMap[app.id] !== 1) return app;
if(! user.installedApps.includes(app.id)) return app;

try {
const appYamlPath = path.join(constants.APP_DATA_DIR, app.id, APP_MANIFEST_FILENAME);
const appYaml = await diskService.readFile(appYamlPath, "utf-8");

const installedApp = YAML.parse(appYaml);
const installedApp = await diskService.readYamlFile(appYamlPath);

app.updateAvailable = installedApp.version != app.version;
} catch(e) {
Expand All @@ -130,16 +158,27 @@ async function get(query) {
return apps;
}

async function find(id){
const user = await diskLogic.readUserFile();

// Loop through the repos to find the app
for (const repoUrl of user.repos) {
const appYamlPath = path.join(constants.REPOS_DIR, reposLogic.slug(repoUrl), id, APP_MANIFEST_FILENAME);
if(await diskLogic.fileExists(appYamlPath)) {
const activeAppRepoFolder = path.join(constants.REPOS_DIR, reposLogic.slug(repoUrl));
return await getAppManifest(activeAppRepoFolder, id, APP_MANIFEST_FILENAME);
}
}

return null;
}

async function isValidAppId(id) {
// TODO: validate id
return true;
return (await find(id)) !== null;
}

async function canInstallOrUpdateApp(id) {
const user = await diskLogic.readUserFile();

const activeAppRepoFolder = path.join(constants.REPOS_DIR, reposLogic.getId(user));
const app = await getAppManifest(activeAppRepoFolder, id, APP_MANIFEST_FILENAME);
const app = await find(id);

// Now check the app's manifest version
return semver.lte(semver.coerce(app.manifestVersion), semver.coerce(APP_MANIFEST_SUPPORTED_VERSION));
Expand Down Expand Up @@ -178,10 +217,6 @@ async function update(id) {
};

async function uninstall(id) {
if(! await isValidAppId(id)) {
throw new NodeError('Invalid app id');
}

try {
await diskLogic.writeSignalFile(`app-uninstall-${id}`);
} catch (error) {
Expand Down
17 changes: 16 additions & 1 deletion logic/auth.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ const crypto = require('crypto');
const { CipherSeed } = require('aezeed');
const iocane = require("iocane");
const diskLogic = require('logic/disk.js');
const appsLogic = require('logic/apps.js');
const reposLogic = require('logic/repos.js');
const diskService = require('services/disk.js');
const lndApiService = require('services/lndApi.js');
const bashService = require('services/bash.js');
Expand Down Expand Up @@ -210,14 +212,27 @@ async function getInfo() {
try {
const user = await diskLogic.readUserFile();

//remove sensitive info
// Append array of repo objects
user.communityAppRepos = (await reposLogic.all(user)).filter(repo => {
return repo.id !== 'umbrel';
});

// Remove sensitive info
delete user.password;
delete user.seed;
user.otpEnabled = Boolean(user.otpUri);
delete user.otpUri;

// Remove other internal properties
delete user.repos;
delete user.appOrigin;
delete user.appRepo;
delete user.installedApps;

return user;
} catch (error) {
console.error(error);

throw new NodeError('Unable to get account info');
}
};
Expand Down
19 changes: 16 additions & 3 deletions logic/disk.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ async function deleteFoldersInDir(directory) {
}

async function fileExists(path) {
return diskService.readJsonFile(path)
return diskService.readUtf8File(path)
.then(() => Promise.resolve(true))
.catch(() => Promise.resolve(false));
}
Expand Down Expand Up @@ -239,14 +239,25 @@ function readDebugStatusFile() {
return diskService.readJsonFile(constants.DEBUG_STATUS_FILE);
}

function readRepoUpdateStatusFile() {
return diskService.readJsonFile(constants.REPO_UPDATE_STATUS_FILE);
}

async function deleteRepoUpdateStatusFile() {
const statusFile = constants.REPO_UPDATE_STATUS_FILE;
if(await fileExists(statusFile)) {
return diskService.deleteFile(constants.REPO_UPDATE_STATUS_FILE);
}
}

// TODO: Transition all logic to use this signal function
function writeSignalFile(signalFile) {
function writeSignalFile(signalFile, contents = 'true') {
if(!/^[0-9a-zA-Z-_]+$/.test(signalFile)) {
throw new Error('Invalid signal file characters');
}

const signalFilePath = path.join(constants.SIGNAL_DIR, signalFile);
return diskService.writeFile(signalFilePath, 'true');
return diskService.writeFile(signalFilePath, contents);
}

// TODO: Transition all logic to use this status function
Expand Down Expand Up @@ -350,6 +361,8 @@ module.exports = {
enableSsh,
readSshSignalFile,
readDebugStatusFile,
readRepoUpdateStatusFile,
deleteRepoUpdateStatusFile,
writeSignalFile,
writeStatusFile,
readHiddenService,
Expand Down
115 changes: 106 additions & 9 deletions logic/repos.js
Original file line number Diff line number Diff line change
@@ -1,16 +1,113 @@
const path = require('path');
const constants = require('utils/const.js');
const diskService = require('services/disk.js');
const diskLogic = require('logic/disk.js');
const NodeError = require('models/errors.js').NodeError;

// Based on a user return the active repo id
function getId(user) {
if(typeof(user.appRepo) !== "string")
{
throw new NodeError("appRepo is not defined within user.json");
}
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));

const REPO_MANIFEST_FILENAME = "umbrel-app-store.yml";

// Given a repo url, return the repo folder name
// Use '-' as a slug
function slug(repoUrl) {
// Replace all non alpha-numeric characters with hyphen
return user.appRepo.replace(/[^a-zA-Z0-9]/g, "-");
return repoUrl.replace(/[^a-zA-Z0-9]/g, "-");
}

function fullUrl(str) {
const repoUrl = str.split('#')[0];

if(str.includes('://') || str.includes('@')) {
return repoUrl;
}

// We'll assume github.com
return `https://github.com/${repoUrl}`;
}

async function transformRepo(repoUrl) {
if(repoUrl === constants.UMBREL_APP_STORE_REPO.url) {
return constants.UMBREL_APP_STORE_REPO;
}

const id = slug(repoUrl);

const repoYamlPath = path.join(constants.REPOS_DIR, id, REPO_MANIFEST_FILENAME);
const repo = await diskService.readYamlFile(repoYamlPath);

repo.url = repoUrl;

return repo;
}

function filterMapFulfilled(list) {
return list.filter(settled => settled.status === 'fulfilled').map(settled => settled.value);
}

async function all(user) {
const repos = await Promise.allSettled(user.repos.map(repoUrl => transformRepo(repoUrl)));

return filterMapFulfilled(repos);
}

async function add(repoUrl) {
try {
await diskLogic.deleteRepoUpdateStatusFile();
} catch (error) {
throw new NodeError('Could not delete repo update status file');
}

try {
await diskLogic.writeSignalFile('repo-add', repoUrl);
} catch (error) {
throw new NodeError('Could not write the signal file');
}

const start = (new Date()).getTime();
const maxWait = 30 * 1000;
while((new Date()).getTime() - start < maxWait) {
let result;
try {
result = await diskLogic.readRepoUpdateStatusFile();
} catch(e) {
// The status file may not exist yet...
console.error(e);

continue;
}

if(result.url === repoUrl) {
if(result.state === 'error') {
throw new NodeError(result.description);
}
else if(result.state === 'success') {
return;
}
else {
// The process is still running...
}
}

await delay(1000);
}

throw new NodeError(`Failed to add: ${repoUrl} as we timed out`);
};

async function remove(repoUrl) {
try {
await diskLogic.writeSignalFile('repo-remove', repoUrl);
} catch (error) {
throw new NodeError('Could not write the signal file');
}
};

module.exports = {
getId
};
slug,
fullUrl,
all,
add,
remove
};

1 change: 1 addition & 0 deletions routes/v1/apps.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ const safeHandler = require('utils/safeHandler');
router.get('/', auth.jwt, safeHandler(async (req, res) => {
const query = {
installed: req.query.installed === '1',
repo: req.query.repo === undefined ? 'umbrel' : req.query.repo
};
const apps = await appsLogic.get(query);

Expand Down
Loading

0 comments on commit eb043b7

Please sign in to comment.