-
Notifications
You must be signed in to change notification settings - Fork 1
/
shell.nix
257 lines (231 loc) · 8.2 KB
/
shell.nix
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
{
pkgs ? import <nixpkgs> {}
#pkgs ? import ./. {}
}:
let
# nix-init ./nix/pyppeteer-stealth.nix --url https://pypi.org/project/pyppeteer-stealth/
# HTTP 404
# expected https://files.pythonhosted.org/packages/f8/64/ae51d6c88406ab8a685b0c83af9fc6ef4275982f391258d9167ddde88cf1/pyppeteer_stealth-2.7.4.tar.gz
# actual https://pypi.org/packages/source/p/pyppeteer-stealth/pyppeteer-stealth-2.7.4.tar.gz
extraPythonPackages = rec {
#pyppeteer-stealth = pkgs.python3.pkgs.callPackage ./nix/pyppeteer-stealth.nix {};
#playwright-stealth = pkgs.python3.pkgs.callPackage ./nix/playwright-stealth.nix {};
#undetected-playwright = pkgs.python3.pkgs.callPackage ./nix/undetected-playwright.nix {};
#pygnuutils = pkgs.python3.pkgs.callPackage ./nix/pygnuutils.nix {};
#pycdlib = pkgs.python3.pkgs.callPackage ./nix/pycdlib.nix {};
chromecontroller = pkgs.python3.pkgs.callPackage ./nix/chromecontroller.nix {};
browser-debugger-tools = pkgs.python3.pkgs.callPackage ./nix/browser-debugger-tools.nix {};
pychrome = pkgs.python3.pkgs.callPackage ./nix/pychrome.nix {};
pychromedevtools = pkgs.python3.pkgs.callPackage ./nix/pychromedevtools.nix {};
cdp-socket = pkgs.python3.pkgs.callPackage ./nix/cdp-socket.nix {};
# error: Package ‘python3.10-selenium-driverless-1.6.3.3’ has an unfree license (‘cc-by-nc-sa-40’), refusing to evaluate.
selenium-driverless = pkgs.python3.pkgs.callPackage ./nix/selenium-driverless.nix {
cdp-socket = pkgs.python3.pkgs.callPackage ./nix/cdp-socket.nix {};
selenium = pkgs.python3.pkgs.callPackage ./nix/selenium.nix { };
};
stream-zip = pkgs.python3.pkgs.callPackage ./nix/stream-zip.nix { };
# fix: ModuleNotFoundError: No module named 'selenium.webdriver.common.devtools'
# https://github.com/milahu/nixpkgs/issues/20
selenium = pkgs.python3.pkgs.callPackage ./nix/selenium.nix { };
# undetected_chromedriver
undetected-chromedriver = pkgs.python3.pkgs.callPackage ./nix/undetected-chromedriver.nix {
# TODO scope
pkgs-undetected-chromedriver = pkgs.callPackage ./nix/undetected-chromedriver-bin.nix {};
};
selenium-wire = pkgs.python3.pkgs.callPackage ./nix/selenium-wire.nix {
# TODO scope
inherit undetected-chromedriver;
};
flaresolverr = pkgs.python3.pkgs.callPackage ./nix/flaresolverr.nix {
# TODO scope
inherit undetected-chromedriver;
};
pysub-parser = pkgs.python3.pkgs.callPackage ./nix/pysub-parser.nix {};
torf = pkgs.python3.pkgs.callPackage ./nix/torf.nix {
flatbencode = pkgs.python3.pkgs.callPackage ./nix/flatbencode.nix {};
};
};
#sqlite-bench = pkgs.callPackage ./nix/sqlite-bench.nix {};
buster-client = pkgs.callPackage ./nix/buster-client.nix {};
buster-client-setup = pkgs.callPackage ./nix/buster-client-setup.nix {
buster-client = pkgs.callPackage ./nix/buster-client.nix {};
};
buster-client-setup-cli = pkgs.callPackage ./nix/buster-client-setup-cli.nix {
buster-client = pkgs.callPackage ./nix/buster-client.nix {};
};
python = pkgs.python3.withPackages (pythonPackages:
(with pythonPackages; [
requests
magic # libmagic
chardet
charset-normalizer
guessit # parse video filenames
langcodes
#playwright
setuptools # pkg_resources for playwright-stealth
#pyppeteer pyppeteer-stealth # puppeteer # old
#kaitaistruct
#sqlglot
# distributed processing
# ray is too complex, has only binary package in nixpkgs https://github.com/NixOS/nixpkgs/pull/194357
#ray
# https://github.com/tomerfiliba-org/rpyc
#rpyc
aiohttp
aiohttp-socks # https://stackoverflow.com/a/76656557/10440128
aiohttp-retry
aiodns # make aiohttp faster
brotli # make aiohttp faster
natsort
#pycdlib
psutil
pyparsing
cryptography
nest-asyncio
# FIXME passlib.exc.InternalBackendError: crypt.crypt() failed for unknown reason; passlib recommends running `pip install bcrypt` for general bcrypt support.(config=<hash <class 'str'> value omitted>, secret=<hash <class 'bytes'> value omitted>)
#bcrypt
# no. use patched version in lib/thirdparty/pysubs2
#pysubs2
lxml # xhtml parser
beautifulsoup4 # html parser
fritzconnection # fritzbox client
#selenium
praw # python reddit api wrapper
])
++
(with extraPythonPackages; [
#pygnuutils # GNU version sort
#playwright-stealth # FIXME not found
#pyppeteer-stealth # FIXME not found
#undetected-playwright # FIXME not found
#chromecontroller
#browser-debugger-tools
#pychrome
#pychromedevtools
selenium-driverless
cdp-socket
/*
selenium-wire
selenium
flaresolverr
undetected-chromedriver
#pysub-parser
*/
selenium
stream-zip
torf # torrent file
])
);
# building sqlite took about 15 minutes on my laptop
sqlite-debug = (pkgs.sqlite.overrideAttrs (oldAttrs: {
#src = ./sqlite;
NIX_CFLAGS_COMPILE = oldAttrs.NIX_CFLAGS_COMPILE + " " + (toString [
# https://sqlite.org/debugging.html
# https://sqlite.org/compile.html
"-DSQLITE_DEBUG=1" # enable assert() statements (run 3x slower), enable debugging of the Virtual Machine
"-DSQLITE_ENABLE_EXPLAIN_COMMENTS=1" # add comment text to the output of EXPLAIN
"-DSQLITE_ENABLE_TREETRACE=1" # .treetrace: trace SELECT and DML statements # not working?
"-DSQLITE_ENABLE_WHERETRACE=1" # .wheretrace: trace WHERE clauses
"-DSQLITE_ENABLE_IOTRACE=1" # .iotrace: low-level log of I/O activity
"-DSQLITE_ENABLE_OFFSET_SQL_FUNC=1" # sqlite_offset(X): get offset in database file
]);
}));
chromium = pkgs.ungoogled-chromium;
in
pkgs.mkShell rec {
#PLAYWRIGHT_BROWSERS_PATH = "${pkgs.playwright.browsers}";
#CHROME_BIN = "${chromium.outPath}/bin/chromium";
# https://github.com/justinwoo/my-blog-posts/blob/master/posts/2019-08-23-using-puppeteer-with-node2nix.md
# https://github.com/puppeteer/puppeteer/issues/244 # Method to skip installing Chromium
#PUPPETEER_SKIP_CHROMIUM_DOWNLOAD = "1";
#PUPPETEER_EXECUTABLE_PATH = "${chromium.outPath}/bin/chromium";
#PYPPETEER_SKIP_CHROMIUM_DOWNLOAD = PUPPETEER_SKIP_CHROMIUM_DOWNLOAD;
#PYPPETEER_EXECUTABLE_PATH = PUPPETEER_EXECUTABLE_PATH;
# used by flaresolverr, fetch-subs.py
# https://github.com/FlareSolverr/FlareSolverr/pull/992
/*
CHROME_EXE_PATH = "${chromium}/bin/${chromium.meta.mainProgram}";
PATCHED_DRIVER_PATH = "${extraPythonPackages.undetected-chromedriver.bin}/bin/chromedriver";
PATCHED_DRIVER_IS_PATCHED = "1";
*/
buildInputs = (with pkgs; [
#gnumake
#playwright
#squashfsTools # mksquashfs
/*
sqlite
udftools # mkudffs
xorriso # xorrisofs
#libfaketime # faketime # this was a desperate attempt at reproducible UDF images
tigervnc # vnc server: Xvnc
xcalib # invert colors: xcalib -i -a
openssh # ssh client
# https://en.wikipedia.org/wiki/Compositing_window_manager
# compositing window managers are not lightweight
# so we just use a compositor
picom
*/
/*
# lightweight window managers
icewm
openbox
tint2 # taskbar (for openbox etc)
# https://wiki.archlinux.org/title/List_of_applications/Other#Taskbars
awesome
fluxbox
spectrwm
qtile
#worm # error: undefined variable 'worm'
dwm
fvwm
dmenu
i3 xss-lock dex networkmanagerapplet i3status
*/
/*
# compressed filesystems
erofs-utils
squashfs-tools-ng
# captcha solver
# https://github.com/dessant/buster
buster-client
#buster-client-setup
buster-client-setup-cli
gost
chromedriver # for selenium
*/
chromium # for selenium
torrenttools
lighttpd
]) ++ [
python
#sqlite-debug
#sqlite-bench
]
++
(with extraPythonPackages; [
#playwright-stealth
#pygnuutils
#pyppeteer-stealth
#undetected-playwright
#pycdlib
/*
chromecontroller
browser-debugger-tools
pychrome
pychromedevtools
*/
selenium-driverless
cdp-socket
/*
selenium-wire
flaresolverr
undetected-chromedriver
chromium
*/
#pysub-parser
# fix: ModuleNotFoundError: No module named 'selenium.webdriver.common.devtools'
selenium
stream-zip
torf # torrent file
]);
}