Skip to content

Commit

Permalink
Merge pull request #739 from podverse/develop
Browse files Browse the repository at this point in the history
Release v4.16.9
  • Loading branch information
mitchdowney authored Apr 19, 2024
2 parents 73b45a4 + bd14fd4 commit f4bbe21
Show file tree
Hide file tree
Showing 7 changed files with 113 additions and 5 deletions.
7 changes: 6 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "podverse-api",
"version": "4.16.8",
"version": "4.16.9",
"description": "Data API, database migration scripts, and backend services for all Podverse models.",
"contributors": [
"Mitch Downey"
Expand Down Expand Up @@ -28,6 +28,7 @@
"dev:scripts:addNonPodcastIndexFeedUrlsToPriorityQueue": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/queue/addNonPodcastIndexFeedUrlsToPriorityQueue.ts",
"dev:scripts:addRecentlyUpdatedFeedUrlsToPriorityQueue": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/podcastIndex/addRecentlyUpdatedFeedUrlsToPriorityQueue.ts",
"dev:scripts:backupDatabase": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/backupDatabase/backupDatabase.ts",
"dev:scripts:devtools:generateDeadRowsCSV": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/devtools/db/generateDeadRowsCSV.ts",
"dev:scripts:dropAndRecreateEpisodesMostRecentMaterializedView": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/episodes/dropAndRecreateEpisodesMostRecentMaterializedView.ts",
"dev:scripts:hideDeadPodcasts": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/podcastIndex/hideDeadPodcasts.ts",
"dev:scripts:parseFeedUrlsByPodcastIds": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/parser/parseFeedUrlsByPodcastIds.ts",
Expand All @@ -43,6 +44,7 @@
"dev:scripts:refreshMediaRefsVideosMaterializedView": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/mediaRefs_videos/refreshMediaRefsVideosMaterializedView.ts",
"dev:scripts:removeDeadChapters": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/mediaRefs/removeDeadChapters.ts",
"dev:scripts:removeDeadEpisodes": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/episodes/removeDeadEpisodes.ts",
"dev:scripts:removePodcastsByIds": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/podcasts/removePodcastsByIds.ts",
"dev:scripts:syncWithFeedUrlsCSVDump": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/podcastIndex/syncWithFeedUrlsCSVDump.ts",
"dev:scripts:updateRecentEpisodesTables": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/scripts/recentEpisodes/updateRecentEpisodesTables.ts",
"dev:seeds:categories": "ts-node -r dotenv/config -r tsconfig-paths/register ./src/seeds/categories.ts",
Expand All @@ -62,6 +64,7 @@
"scripts:addNonPodcastIndexFeedUrlsToPriorityQueue": "ts-node -r tsconfig-paths/register ./src/scripts/queue/addNonPodcastIndexFeedUrlsToPriorityQueue.ts",
"scripts:addRecentlyUpdatedFeedUrlsToPriorityQueue": "ts-node -r tsconfig-paths/register ./src/scripts/podcastIndex/addRecentlyUpdatedFeedUrlsToPriorityQueue.ts",
"scripts:backupDatabase": "ts-node -r tsconfig-paths/register ./src/scripts/backupDatabase/backupDatabase.ts",
"scripts:devtools:generateDeadRowsCSV": "ts-node -r tsconfig-paths/register ./src/scripts/devtools/db/generateDeadRowsCSV.ts",
"scripts:dropAndRecreateEpisodesMostRecentMaterializedView": "ts-node -r tsconfig-paths/register ./src/scripts/episodes/dropAndRecreateEpisodesMostRecentMaterializedView.ts",
"scripts:hideDeadPodcasts": "ts-node -r tsconfig-paths/register ./src/scripts/podcastIndex/hideDeadPodcasts.ts",
"scripts:parseFeedUrlsByPodcastIds": "ts-node -r tsconfig-paths/register ./src/scripts/parser/parseFeedUrlsByPodcastIds.ts",
Expand All @@ -76,6 +79,7 @@
"scripts:refreshEpisodesMostRecentMaterializedView": "ts-node -r tsconfig-paths/register ./src/scripts/episodes/refreshEpisodesMostRecentMaterializedView.ts",
"scripts:refreshMediaRefsVideosMaterializedView": "ts-node -r tsconfig-paths/register ./src/scripts/mediaRefs_videos/refreshMediaRefsVideosMaterializedView.ts",
"scripts:removeDeadEpisodes": "ts-node -r tsconfig-paths/register ./src/scripts/episodes/removeDeadEpisodes.ts",
"scripts:removePodcastsByIds": "ts-node -r tsconfig-paths/register ./src/scripts/podcasts/removePodcastsByIds.ts",
"scripts:syncWithFeedUrlsCSVDump": "ts-node -r tsconfig-paths/register ./src/scripts/podcastIndex/syncWithFeedUrlsCSVDump.ts",
"scripts:updateRecentEpisodesTables": "ts-node -r tsconfig-paths/register ./src/scripts/recentEpisodes/updateRecentEpisodesTables.ts",
"seeds:categories": "ts-node -r tsconfig-paths/register ./src/seeds/categories.ts",
Expand Down Expand Up @@ -177,6 +181,7 @@
"isomorphic-fetch": "^3.0.0",
"jest": "26.1.0",
"joi": "14.3.1",
"json-2-csv": "^5.5.1",
"koa": "2.11.0",
"koa-bodyparser": "4.2.1",
"koa-helmet": "5.2.0",
Expand Down
20 changes: 20 additions & 0 deletions src/controllers/devtools/db.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import { getConnection } from 'typeorm'
import { _logEnd, _logStart, logPerformance } from '~/lib/utility'
const json2csv = require('json-2-csv')

const generateDeadRowsCSV = async () => {
logPerformance('generateDeadRowCSV', _logStart)
const em = await getConnection().createEntityManager()
const results = await em.query(`
SELECT n_live_tup, n_dead_tup, relname
FROM pg_stat_all_tables
ORDER BY n_dead_tup DESC;
`)

const csv = json2csv.json2csv(results)
console.log('csv', csv)

logPerformance('generateDeadRowCSV', _logEnd)
}

export { generateDeadRowsCSV }
27 changes: 25 additions & 2 deletions src/controllers/podcast.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,14 @@ import { getRepository, In } from 'typeorm'
import { config } from '~/config'
import { getUserSubscribedPodcastIds } from '~/controllers/user'
import { FeedUrl, Podcast, User } from '~/entities'
import { addOrderByToQuery, getManticoreOrderByColumnName, removeAllSpaces } from '~/lib/utility'
import {
_logEnd,
_logStart,
addOrderByToQuery,
getManticoreOrderByColumnName,
logPerformance,
removeAllSpaces
} from '~/lib/utility'
import { validateSearchQueryString } from '~/lib/utility/validation'
import { manticoreWildcardSpecialCharacters, searchApi } from '~/services/manticore'
import { deleteNotification } from './notification'
Expand Down Expand Up @@ -506,6 +513,21 @@ const getPodcastWebUrl = async ({ podcastGuid, podcastIndexId }: GetPodcastWebUr
}
}

const removePodcasts = async (ids: string[]) => {
logPerformance('removePodcasts', _logStart)
const repository = getRepository(Podcast)
for (const id of ids) {
try {
logPerformance(`removePodcast ${id}`, _logStart)
await repository.delete(id)
logPerformance(`removePodcast ${id}`, _logEnd)
} catch (error) {
console.log(`error removing podcast ${id}`, error)
}
}
logPerformance('removePodcasts', _logEnd)
}

export {
findPodcastsByFeedUrls,
getPodcast,
Expand All @@ -519,5 +541,6 @@ export {
subscribeToPodcast,
toggleSubscribeToPodcast,
updateHasPodcastIndexValueTags,
getPodcastWebUrl
getPodcastWebUrl,
removePodcasts
}
4 changes: 2 additions & 2 deletions src/routes/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ router.post('/findHTTPS', async (ctx) => {

const extractHTTPSFromURL = async (url, tries) => {
try {
const res = await request(url, { followRedirect: false, method: 'head' })
const res = await request(url, { followRedirect: true, method: 'head' })
if (tries > 5) {
throw new createError.NotFound('Secure URL for ' + url + ' was not found. Too many redirects')
} else if (!res.location) {
Expand All @@ -35,7 +35,7 @@ const extractHTTPSFromURL = async (url, tries) => {
try {
const attemptHttpsUrl = url.replace('http://', 'https://')
// If no error is thrown,then assume it is a valid url.
await request(attemptHttpsUrl, { followRedirect: false, method: 'head' })
await request(attemptHttpsUrl, { followRedirect: true, method: 'head' })
return attemptHttpsUrl
} catch (error) {
throw new createError.NotFound('Secure URL for ' + url + ' was not found.')
Expand Down
17 changes: 17 additions & 0 deletions src/scripts/devtools/db/generateDeadRowsCSV.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import { generateDeadRowsCSV as generateDeadRowsCSVCont } from '~/controllers/devtools/db'
import { connectToDb } from '~/lib/db'
;(async function () {
const generateDeadRowCSV = async () => {
try {
const connection = await connectToDb()

if (connection && connection.isConnected) {
await generateDeadRowsCSVCont()
}
} catch (error) {
console.log(error)
}
}

generateDeadRowCSV()
})()
25 changes: 25 additions & 0 deletions src/scripts/podcasts/removePodcastsByIds.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import { removePodcasts } from '~/controllers/podcast'
import { connectToDb } from '~/lib/db'
;(async function () {
const removePodcastsByIds = async () => {
try {
const podcastIdsString = process.argv.length > 2 ? process.argv[2] : ''

if (!podcastIdsString) {
console.log('Provide a comma delimited string of podcastIds as a parameter.')
return
}

const podcastIds = podcastIdsString.split(',')
const connection = await connectToDb()

if (connection && connection.isConnected) {
await removePodcasts(podcastIds)
}
} catch (error) {
console.log(error)
}
}

removePodcastsByIds()
})()
18 changes: 18 additions & 0 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -3227,6 +3227,11 @@ dedent@^0.7.0:
resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c"
integrity sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA==

deeks@3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/deeks/-/deeks-3.1.0.tgz#ecc47c600bcc6dbda656c6d86bbc96cbe126fd41"
integrity sha512-e7oWH1LzIdv/prMQ7pmlDlaVoL64glqzvNgkgQNgyec9ORPHrT2jaOqMtRyqJuwWjtfb6v+2rk9pmaHj+F137A==

deep-eql@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-3.0.1.tgz#dfc9404400ad1c8fe023e7da1df1c147c4b444df"
Expand Down Expand Up @@ -3404,6 +3409,11 @@ dir-glob@^3.0.1:
dependencies:
path-type "^4.0.0"

doc-path@4.1.1:
version "4.1.1"
resolved "https://registry.yarnpkg.com/doc-path/-/doc-path-4.1.1.tgz#5be8c1671877f6b719af5a6077c41904be8913ac"
integrity sha512-h1ErTglQAVv2gCnOpD3sFS6uolDbOKHDU1BZq+Kl3npPqroU3dYL42lUgMfd5UimlwtRgp7C9dLGwqQ5D2HYgQ==

docker-cli-js@2.9.0:
version "2.9.0"
resolved "https://registry.yarnpkg.com/docker-cli-js/-/docker-cli-js-2.9.0.tgz#1584461134cbb6beb548fed2c33de8f5593280bf"
Expand Down Expand Up @@ -6276,6 +6286,14 @@ jsesc@^2.5.1:
resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4"
integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==

json-2-csv@^5.5.1:
version "5.5.1"
resolved "https://registry.yarnpkg.com/json-2-csv/-/json-2-csv-5.5.1.tgz#2ad4e69d4b2bb4590203948a36f11347a4d7aed9"
integrity sha512-KgAtAXTQopRwe90gh8SgjRSxgt9bUWbGAPMo9W0TZLA8SqiQH7khtagFfeEUjG3NBPwJu/+9uX5pMvunKaPvrQ==
dependencies:
deeks "3.1.0"
doc-path "4.1.1"

json-bigint@^0.3.0:
version "0.3.1"
resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-0.3.1.tgz#0c1729d679f580d550899d6a2226c228564afe60"
Expand Down

0 comments on commit f4bbe21

Please sign in to comment.