add proof of origin token (poToken) support

Support poToken/visitorData challenge, based on: https://github.com/yt-dlp/yt-dlp/wiki/Extractors#po-token-guide yt-dlp/yt-dlp#10648 iv-org/invidious#4789 By following the steps in the yt-dlp wiki page (linked above), I was able to get a working (poToken, visitorData) pair in Chromium. For some reason, I could not get a working pair in Firefox Nightly; maybe my profile in the latter is fingerprinted and soft-blocked somehow?
etwoo · Oct 13, 2024 · fe9d033 · fe9d033
1 parent 6703516
commit fe9d033
Show file tree

Hide file tree

Showing 11 changed files with 167 additions and 56 deletions.
diff --git a/include/result.h b/include/result.h
@@ -76,6 +76,9 @@ typedef struct {
 		ERR_YOUTUBE_N_PARAM_FIND_IN_QUERY,
 		ERR_YOUTUBE_N_PARAM_KVPAIR_ALLOC,
 		ERR_YOUTUBE_N_PARAM_QUERY_APPEND_PLAINTEXT,
+		ERR_YOUTUBE_POT_PARAM_KVPAIR_ALLOC,
+		ERR_YOUTUBE_POT_PARAM_QUERY_APPEND_PLAINTEXT,
+		ERR_YOUTUBE_VISITOR_DATA_HEADER_ALLOC,
 		ERR_YOUTUBE_STREAM_VISITOR_GET_URL,
 	} err;
 	int num; /* may hold errno, CURLcode, CURLUcode, etc */

diff --git a/include/youtube.h b/include/youtube.h
@@ -9,7 +9,8 @@ void youtube_global_cleanup(void);
 
 typedef struct youtube_stream *youtube_handle_t;
 
-youtube_handle_t youtube_stream_init(void) WARN_UNUSED;
+youtube_handle_t youtube_stream_init(const char *proof_of_origin,
+                                     const char *visitor_data) WARN_UNUSED;
 void youtube_stream_cleanup(youtube_handle_t h);
 
 struct youtube_setup_ops {

diff --git a/main.c b/main.c
@@ -15,6 +15,7 @@
 #include "youtube.h"
 
 #include <assert.h>
+#include <getopt.h> /* for getopt_long() */
 #include <stdarg.h>
 #include <stdbool.h>
 #include <stdio.h>
@@ -24,10 +25,6 @@
 #define PCRE2_CODE_UNIT_WIDTH 8
 #include <pcre2.h>
 
-static const char ARG_HELP[] = "--help";
-static const char ARG_SANDBOX[] = "--try-sandbox";
-static const char ARG_QUALITY[] = "--quality=";
-
 const char *__asan_default_options(void) __attribute__((used));
 
 const char *
@@ -194,26 +191,59 @@ print_url(const char *url)
 	} while (0)
 
 int
-main(int argc, const char *argv[])
+main(int argc, char *argv[])
 {
 	int fd __attribute__((cleanup(coverage_cleanup))) = coverage_open();
 
-	if (argc < 2) {
+	struct quality q = {NULL, NULL};
+	const char *proof_of_origin = NULL;
+	const char *visitor_data = NULL;
+
+	int synonym = 0;
+	struct option lo[] = {
+		{"help", no_argument, &synonym, 'h'},
+		{"try-sandbox", no_argument, &synonym, 't'},
+		{"quality", required_argument, &synonym, 'q'},
+		{"proof-of-origin", required_argument, &synonym, 'p'},
+		{"visitor-data", required_argument, &synonym, 'v'},
+	};
+
+	int opt = 0;
+	while ((opt = getopt_long(argc, argv, "htq:p:v:", lo, NULL)) != -1) {
+		switch (opt == 0 ? synonym : opt) {
+		case 'h':
+			return usage(argv[0], EX_OK);
+		case 't':
+			return try_sandbox();
+		case 'q':
+			if (!parse_quality_choices(optarg, &q)) {
+				return EX_DATAERR;
+			}
+			break;
+		case 'p':
+			proof_of_origin = optarg;
+			break;
+		case 'v':
+			visitor_data = optarg;
+			break;
+		default:
+			return usage(argv[0], EX_USAGE);
+		}
+	}
+
+	if (optind >= argc) {
+		fprintf(stderr, "Expected URL argument after options\n");
 		return usage(argv[0], EX_USAGE);
 	}
 
-	int idx = 1;
-	const char *arg1 = argv[idx];
-	struct quality q = {NULL, NULL};
-	if (0 == strncmp(ARG_HELP, arg1, strlen(ARG_HELP))) {
-		return usage(argv[0], EX_OK);
-	} else if (0 == strncmp(ARG_SANDBOX, arg1, strlen(ARG_SANDBOX))) {
-		return try_sandbox();
-	} else if (0 == strncmp(ARG_QUALITY, arg1, strlen(ARG_QUALITY))) {
-		if (!parse_quality_choices(arg1 + strlen(ARG_QUALITY), &q)) {
-			return EX_DATAERR;
-		}
-		++idx;
+	if (proof_of_origin == NULL || strlen(proof_of_origin) == 0) {
+		fprintf(stderr, "Expected --proof-of-origin value\n");
+		return usage(argv[0], EX_USAGE);
+	}
+
+	if (visitor_data == NULL || strlen(visitor_data) == 0) {
+		fprintf(stderr, "Expected --visitor-data value\n");
+		return usage(argv[0], EX_USAGE);
 	}
 
 	int rc = EX_OK;
@@ -222,7 +252,7 @@ main(int argc, const char *argv[])
 	check_stderr(youtube_global_init(), EX_SOFTWARE);
 	check_stderr(sandbox_only_io_inet_tmpfile(), EX_OSERR);
 
-	stream = youtube_stream_init();
+	stream = youtube_stream_init(proof_of_origin, visitor_data);
 	if (stream == NULL) {
 		fprintf(stderr, "ERROR: Cannot allocate stream object\n");
 		rc = EX_OSERR;
@@ -241,7 +271,7 @@ main(int argc, const char *argv[])
 		.after = NULL,
 	};
 
-	const char *url = argv[idx];
+	const char *url = argv[optind];
 	check_stderr(youtube_stream_setup(stream, &sops, &q, url), EX_DATAERR);
 	check_stderr(youtube_stream_visitor(stream, print_url), EX_DATAERR);
 

diff --git a/src/js.c b/src/js.c
@@ -142,6 +142,7 @@ parse_json(const char *json, size_t json_sz, struct parse_ops *ops)
 
 result_t
 make_innertube_json(const char *target_url,
+                    const char *proof_of_origin,
                     long long int timestamp,
                     char **body)
 {
@@ -159,13 +160,13 @@ make_innertube_json(const char *target_url,
 	debug("Parsed ID: %.*s", (int)sz, id);
 
 	json_auto_t *obj = NULL;
-	obj = json_pack("{s{s{ss,ss,ss,ss,si}},ss%,s{s{ss,si}},sb,sb}",
+	obj = json_pack("{s{s{ss,ss,ss,ss,si}},ss%,s{ss},s{s{ss,si}},sb,sb}",
 	                "context",
 	                "client",
 	                "clientName",
-	                "WEB_CREATOR",
+	                "WEB",
 	                "clientVersion",
-	                "1.20240723.03.00",
+	                "2.20240726.00.00",
 	                "hl",
 	                "en",
 	                "timeZone",
@@ -175,6 +176,9 @@ make_innertube_json(const char *target_url,
 	                "videoId",
 	                id,
 	                sz,
+	                "serviceIntegrityDimensions",
+	                "poToken",
+	                proof_of_origin,
 	                "playbackContext",
 	                "contentPlaybackContext",
 	                "html5Preference",

diff --git a/src/js.h b/src/js.h
@@ -20,6 +20,7 @@ result_t parse_json(const char *json_text,
                     struct parse_ops *ops) WARN_UNUSED;
 
 result_t make_innertube_json(const char *target_url,
+                             const char *proof_of_origin,
                              long long int timestamp,
                              char **body);
 

diff --git a/src/result.c b/src/result.c
@@ -313,12 +313,22 @@ result_to_str(result_t r)
 		my_snprintf("No n-parameter in query string: %s", r.msg);
 		break;
 	case ERR_YOUTUBE_N_PARAM_KVPAIR_ALLOC:
-		s = "Cannot allocate kv-pair buffer";
+		s = "Cannot allocate kv-pair buffer for plaintext n-parameter";
 		break;
 	case ERR_YOUTUBE_N_PARAM_QUERY_APPEND_PLAINTEXT:
 		my_snprintf("Cannot append plaintext n-parameter: %s",
 		            url_error(r));
 		break;
+	case ERR_YOUTUBE_POT_PARAM_KVPAIR_ALLOC:
+		s = "Cannot allocate kv-pair buffer for proof of origin";
+		break;
+	case ERR_YOUTUBE_POT_PARAM_QUERY_APPEND_PLAINTEXT:
+		my_snprintf("Cannot append proof of origin parameter: %s",
+		            url_error(r));
+		break;
+	case ERR_YOUTUBE_VISITOR_DATA_HEADER_ALLOC:
+		s = "Cannot allocate asprintf buffer for visitor data header";
+		break;
 	case ERR_YOUTUBE_STREAM_VISITOR_GET_URL:
 		my_snprintf("Cannot get URL as string: %s", url_error(r));
 		break;

diff --git a/src/url.c b/src/url.c
@@ -71,6 +71,7 @@ url_global_init(void)
 	                            NULL,
 	                            NULL,
 	                            NULL,
+	                            NULL,
 	                            FD_DISCARD);
 	info_if(err.err, "Error creating early URL worker threads");
 
@@ -126,10 +127,11 @@ static const char CONTENT_TYPE_JSON[] = "Content-Type: application/json";
 static const char DEFAULT_HOST_STR[] = "www.youtube.com";
 
 result_t
-url_download(const char *url_str,   /* may be NULL */
-             const char *host_str,  /* may be NULL */
-             const char *path_str,  /* may be NULL */
-             const char *post_body, /* may be NULL */
+url_download(const char *url_str,     /* may be NULL */
+             const char *host_str,    /* may be NULL */
+             const char *path_str,    /* may be NULL */
+             const char *post_body,   /* may be NULL */
+             const char *post_header, /* may be NULL */
              int fd)
 {
 	CURLU *url = NULL;
@@ -170,14 +172,21 @@ url_download(const char *url_str,   /* may be NULL */
 
 	if (post_body) {
 		headers = curl_slist_append(headers, CONTENT_TYPE_JSON);
-		res = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
-		check_if_num(res, ERR_URL_DOWNLOAD_SET_OPT_HTTP_HEADER);
 
 		res = curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_body);
 		/* Note: libcurl does not copy <post_body> */
 		check_if_num(res, ERR_URL_DOWNLOAD_SET_OPT_POST_BODY);
 	}
 
+	if (post_header) {
+		headers = curl_slist_append(headers, post_header);
+	}
+
+	if (headers) {
+		res = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+		check_if_num(res, ERR_URL_DOWNLOAD_SET_OPT_HTTP_HEADER);
+	}
+
 	res = CURL_EASY_PERFORM(curl, url_fragment_or_path_str, fd);
 	check_if_num(res, ERR_URL_DOWNLOAD_PERFORM);
 

diff --git a/src/url.h b/src/url.h
@@ -12,6 +12,7 @@ result_t url_download(const char *url,
                       const char *host,
                       const char *path,
                       const char *post_body,
+                      const char *post_header,
                       int fd) WARN_UNUSED;
 
 #endif