diff --git a/static/build_files/docker/client/Dockerfile b/static/build_files/docker/client/Dockerfile new file mode 100644 index 00000000..a91a47a0 --- /dev/null +++ b/static/build_files/docker/client/Dockerfile @@ -0,0 +1,50 @@ +FROM ghcr.io/suika/opencv-video-minimal:4.5-py3.8 + +ARG UID +ARG GID + +HEALTHCHECK --interval=20s --timeout=10s --retries=3 --start-period=30s CMD ! supervisorctl status | grep -v RUNNING +ENTRYPOINT ["/bin/sh", "/opt/hydrus/static/build_files/docker/client/entrypoint.sh"] +LABEL git="https://github.com/hydrusnetwork/hydrus" + +RUN apk --no-cache add jq fvwm x11vnc xvfb supervisor py3-beautifulsoup4 py3-psutil py3-pysocks py3-requests py3-twisted py3-yaml qt5-qtcharts py3-lz4 ffmpeg py3-pillow py3-numpy py3-numpy py3-qt5 py3-openssl openssl mpv mpv-libs nodejs patch \ + && apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community font-noto font-noto-emoji \ + && apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community font-noto-cjk +RUN pip install qtpy Send2Trash html5lib twisted python-mpv cloudscrape cloudscraper pyparsing + +RUN set -xe \ + && mkdir -p /opt/hydrus \ + && addgroup -g 1000 hydrus \ + && adduser -h /opt/hydrus -u 1000 -H -S -G hydrus hydrus + +RUN mkdir -p /opt/noVNC/utils/websockify \ + && wget $(wget https://api.github.com/repos/novnc/noVNC/releases/latest -qO- | jq -r '.tarball_url') -qO- | tar xzf - --strip-components=1 -C /opt/noVNC \ + && wget $(wget https://api.github.com/repos/novnc/websockify/releases/latest -qO- | jq -r '.tarball_url') -qO- | tar xzf - --strip-components=1 -C /opt/noVNC/utils/websockify \ + && sed -i -- "s/ps -p/ps -o pid | grep/g" /opt/noVNC/utils/launch.sh \ + && chown hydrus:hydrus -R /opt/noVNC + +COPY --chown=hydrus . /opt/hydrus +COPY --chown=hydrus --from=suika/swftools:2013-04-09-1007 /swftools/swfrender /opt/hydrus/bin/swfrender_linux + +RUN mv /opt/hydrus/static/build_files/docker/client/supervisord.conf /etc/supervisord.conf && \ + mv /opt/hydrus/static/build_files/docker/client/novnc/index.html /opt/noVNC/index.html && \ + mv /opt/hydrus/static/build_files/docker/client/novnc/icon.png /opt/noVNC/app/images/icons/icon.png + +RUN ln -fs /usr/bin/python3 /usr/bin/python && ln -fs /usr/bin/pip3 /usr/bin/pip + +VOLUME /opt/hydrus/db + +ENV QT_SCALE_FACTOR=1.1 \ + VNC_PORT=5900 \ + NOVNC_PORT=5800 \ + SUPERVISOR_PORT=9001 \ + XVFBRES=1680x1050x24 \ + UID=${UID:-1000} \ + GID=${GID:-1000} \ + DB_DIR=/opt/hydrus/db \ + XVFB_EXTRA="" \ + VNC_EXTRA="" \ + NOVNC_EXTRA="" \ + HYDRUS_EXTRA="" + +EXPOSE 5800 5900 diff --git a/static/build_files/docker/client/entrypoint.sh b/static/build_files/docker/client/entrypoint.sh new file mode 100644 index 00000000..ebc1b7ae --- /dev/null +++ b/static/build_files/docker/client/entrypoint.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +USER_ID=${UID} +GROUP_ID=${GID} + +echo "Starting Hydrus with UID/GID : $USER_ID/$GROUP_ID" + +cd /opt/hydrus/ + +if [ -f "/opt/hydrus/static/build_files/docker/client/patch.patch" ]; then + echo "Patching Hydrus" + patch -f -p1 -i /opt/hydrus/static/build_files/docker/client/patch.patch +fi + +if [ -f "/opt/hydrus/static/build_files/docker/client/requests.patch" ]; then + cd /usr/lib/python3.8/site-packages/requests + echo "Patching Requests" + patch -f -p2 -i /opt/hydrus/static/build_files/docker/client/requests.patch + cd /opt/hydrus/ +fi + +#if [ $USER_ID != 0 ] && [ $GROUP_ID != 0 ]; then +# find /opt/hydrus/ -not -path "/opt/hydrus/db/*" -exec chown hydrus:hydrus "{}" \; +#fi + +exec supervisord -c /etc/supervisord.conf diff --git a/static/build_files/docker/client/novnc/icon.png b/static/build_files/docker/client/novnc/icon.png new file mode 100644 index 00000000..91431202 Binary files /dev/null and b/static/build_files/docker/client/novnc/icon.png differ diff --git a/static/build_files/docker/client/novnc/index.html b/static/build_files/docker/client/novnc/index.html new file mode 100644 index 00000000..da570754 --- /dev/null +++ b/static/build_files/docker/client/novnc/index.html @@ -0,0 +1,328 @@ + + + + + + Hydrus + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
noVNC encountered an error:
+
+
+
+
+ + +
+ +
+
+ +
+ +

no
VNC

+ + + + + +
+ +
+ + + +
+
+ + + + + + +
+
+ + + +
+
+
+ Power +
+ + + +
+
+ + + +
+
+
+ Clipboard +
+ +
+ +
+
+ + + + + + +
+
+
    +
  • + Settings +
    • +
  • + +
    • +
  • + +
    • +
    • +
  • + +
    • +
  • + + +
    • +
    • +
  • +
    Advanced
    +
      +
    • + + +
      • +
    • + + +
      • +
      • +
    • + + +
      • +
    • +
      WebSocket
      +
        +
      • + +
        • +
      • + + +
        • +
      • + + +
        • +
      • + + +
        • +
      +
      • +
      • +
    • + +
      • +
    • + + +
      • +
      • +
    • + +
      • +
      • + +
    • + +
      • +
    +
    • +
    • +
  • + Version: + +
    • +
+
+
+ + + + +
+
+ +
+ +
+ + +
+ + +
+
+ +
+ Connect +
+
+
+ + +
+
+
    +
  • + + +
    • +
  • + + +
    • +
  • + +
    • +
+
+
+ + +
+
+
+ +
+ +
+ + +
+ + +
+ + + + diff --git a/static/build_files/docker/client/requests.patch b/static/build_files/docker/client/requests.patch new file mode 100644 index 00000000..73549e8a --- /dev/null +++ b/static/build_files/docker/client/requests.patch @@ -0,0 +1,189 @@ +From 063f2ae0e67111467fc21a2498e426e42f8fae4b Mon Sep 17 00:00:00 2001 +From: suika <2320837+Suika@users.noreply.github.com> +Date: Thu, 24 Sep 2020 15:49:53 +0200 +Subject: [PATCH 1/4] Bypass proxy if no_proxy or no are set, merge proxy and + self.proxy + +Check if the proxy should be bypassed in case no_proxy or no match the host of the url. +Also, since proxy can be defined in the session and the reuquest itself, both of them were never merged. Now they self.proxies will be updated by proxies. +--- + requests/sessions.py | 22 +++++++++++++++++++++- + 1 file changed, 21 insertions(+), 1 deletion(-) + +diff --git a/requests/sessions.py b/requests/sessions.py +index fdf7e9fe35..bdaf515a65 100644 +--- a/requests/sessions.py ++++ b/requests/sessions.py +@@ -529,6 +529,14 @@ def request(self, method, url, + + proxies = proxies or {} + ++ # Update self.proxy with proxy and assing the result to proxies ++ if isinstance(proxies,dict): ++ slef_proxies_tmp = self.proxies.copy() ++ slef_proxies_tmp.update(proxies) ++ proxies = slef_proxies_tmp.copy() ++ else: ++ proxies = self.proxies.copy() ++ + settings = self.merge_environment_settings( + prep.url, proxies, stream, verify, cert + ) +@@ -705,6 +713,7 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + :rtype: dict + """ + # Gather clues from the surrounding environment. ++ bypass_proxy = False + if self.trust_env: + # Set environment's proxies. + no_proxy = proxies.get('no_proxy') if proxies is not None else None +@@ -712,6 +721,14 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + for (k, v) in env_proxies.items(): + proxies.setdefault(k, v) + ++ # Check for no_proxy and no since they could be loaded from environment ++ no_proxy = proxies.get('no_proxy') if proxies is not None else None ++ no = proxies.get('no') if proxies is not None else None ++ if any([no_proxy,no]): ++ no_proxy = ','.join(filter(None, (no_proxy, no))) ++ if should_bypass_proxies(url, no_proxy): ++ bypass_proxy = True ++ + # Look for requests environment configuration and be compatible + # with cURL. + if verify is True or verify is None: +@@ -719,7 +736,10 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + os.environ.get('CURL_CA_BUNDLE')) + + # Merge all the kwargs. +- proxies = merge_setting(proxies, self.proxies) ++ if bypass_proxy: ++ proxies = {} ++ else: ++ proxies = merge_setting(proxies, self.proxies) + stream = merge_setting(stream, self.stream) + verify = merge_setting(verify, self.verify) + cert = merge_setting(cert, self.cert) + +From a3afa6b55d7596ab6a06207a32a5c96436a66e8c Mon Sep 17 00:00:00 2001 +From: suika <2320837+Suika@users.noreply.github.com> +Date: Thu, 24 Sep 2020 16:00:30 +0200 +Subject: [PATCH 2/4] Rename bypass_proxy to bypass_proxies + +Make it match the variable being returned +--- + requests/sessions.py | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/requests/sessions.py b/requests/sessions.py +index bdaf515a65..2db637dd6e 100644 +--- a/requests/sessions.py ++++ b/requests/sessions.py +@@ -713,7 +713,7 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + :rtype: dict + """ + # Gather clues from the surrounding environment. +- bypass_proxy = False ++ bypass_proxies = False + if self.trust_env: + # Set environment's proxies. + no_proxy = proxies.get('no_proxy') if proxies is not None else None +@@ -727,7 +727,7 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + if any([no_proxy,no]): + no_proxy = ','.join(filter(None, (no_proxy, no))) + if should_bypass_proxies(url, no_proxy): +- bypass_proxy = True ++ bypass_proxies = True + + # Look for requests environment configuration and be compatible + # with cURL. +@@ -736,7 +736,7 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + os.environ.get('CURL_CA_BUNDLE')) + + # Merge all the kwargs. +- if bypass_proxy: ++ if bypass_proxies: + proxies = {} + else: + proxies = merge_setting(proxies, self.proxies) + +From 78682f9e21933bc6defca8f236b6c0bde5ac045f Mon Sep 17 00:00:00 2001 +From: suika <2320837+Suika@users.noreply.github.com> +Date: Thu, 24 Sep 2020 16:12:39 +0200 +Subject: [PATCH 3/4] Move no_proxy check outside trust_env + +It makes more sense to have the check be outside the trust_env. Since it has to be always executed. Because proxy configuration can be performed on the Sessions class. +--- + requests/sessions.py | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/requests/sessions.py b/requests/sessions.py +index 2db637dd6e..178ca7e9a7 100644 +--- a/requests/sessions.py ++++ b/requests/sessions.py +@@ -721,20 +721,20 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + for (k, v) in env_proxies.items(): + proxies.setdefault(k, v) + +- # Check for no_proxy and no since they could be loaded from environment +- no_proxy = proxies.get('no_proxy') if proxies is not None else None +- no = proxies.get('no') if proxies is not None else None +- if any([no_proxy,no]): +- no_proxy = ','.join(filter(None, (no_proxy, no))) +- if should_bypass_proxies(url, no_proxy): +- bypass_proxies = True +- + # Look for requests environment configuration and be compatible + # with cURL. + if verify is True or verify is None: + verify = (os.environ.get('REQUESTS_CA_BUNDLE') or + os.environ.get('CURL_CA_BUNDLE')) + ++ # Check for no_proxy and no since they could be loaded from environment ++ no_proxy = proxies.get('no_proxy') if proxies is not None else None ++ no = proxies.get('no') if proxies is not None else None ++ if any([no_proxy, no]): ++ no_proxy = ','.join(filter(None, (no_proxy, no))) ++ if should_bypass_proxies(url, no_proxy): ++ bypass_proxy = True ++ + # Merge all the kwargs. + if bypass_proxies: + proxies = {} + +From 0f6bd04349dc1bb2c0808f4de8583eace7c5aaa3 Mon Sep 17 00:00:00 2001 +From: suika <2320837+Suika@users.noreply.github.com> +Date: Thu, 24 Sep 2020 16:34:20 +0200 +Subject: [PATCH 4/4] Remove bypass_proxies var and only use + should_bypass_proxies + +That logic was left from previous tires fixing no_proxy and since it's quite compact now, it can be removed and should_bypass_proxies should be used instead of setting a var. +--- + requests/sessions.py | 5 +---- + 1 file changed, 1 insertion(+), 4 deletions(-) + +diff --git a/requests/sessions.py b/requests/sessions.py +index 178ca7e9a7..4ac01315cb 100644 +--- a/requests/sessions.py ++++ b/requests/sessions.py +@@ -713,7 +713,6 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + :rtype: dict + """ + # Gather clues from the surrounding environment. +- bypass_proxies = False + if self.trust_env: + # Set environment's proxies. + no_proxy = proxies.get('no_proxy') if proxies is not None else None +@@ -732,11 +731,9 @@ def merge_environment_settings(self, url, proxies, stream, verify, cert): + no = proxies.get('no') if proxies is not None else None + if any([no_proxy, no]): + no_proxy = ','.join(filter(None, (no_proxy, no))) +- if should_bypass_proxies(url, no_proxy): +- bypass_proxy = True + + # Merge all the kwargs. +- if bypass_proxies: ++ if should_bypass_proxies(url, no_proxy): + proxies = {} + else: + proxies = merge_setting(proxies, self.proxies) \ No newline at end of file diff --git a/static/build_files/docker/client/supervisord.conf b/static/build_files/docker/client/supervisord.conf new file mode 100644 index 00000000..f44532a9 --- /dev/null +++ b/static/build_files/docker/client/supervisord.conf @@ -0,0 +1,58 @@ +[unix_http_server] +file=/run/supervisor.sock + +[inet_http_server] +port=127.0.0.1:%(ENV_SUPERVISOR_PORT)s + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///run/supervisor.sock + +[supervisord] +nodaemon=true +[program:xvfb] +command=Xvfb :89 -ac -listen tcp -screen 0 %(ENV_XVFBRES)s %(ENV_XVFB_EXTRA)s +startretries=89 +autostart=true +autorestart=true + +[program:fvwm] +command=fvwm -d :89 +startretries=89 +autostart=true +autorestart=true + +[program:vnc] +command=x11vnc -display :89 -forever -noxrecord -noxfixes -noxdamage -rfbport %(ENV_VNC_PORT)s %(ENV_VNC_EXTRA)s +startretries=89 +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stderr_logfile=/dev/stderr +stdout_logfile_maxbytes=0 +stderr_logfile_maxbytes=0 + +[program:novnc] +command=sh /opt/noVNC/utils/launch.sh --vnc localhost:%(ENV_VNC_PORT)s --listen %(ENV_NOVNC_PORT)s %(ENV_NOVNC_EXTRA)s +startretries=89 +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stderr_logfile=/dev/stderr +stdout_logfile_maxbytes=0 +stderr_logfile_maxbytes=0 + +[program:hydrus] +environment=DISPLAY=":89",HOME=/opt/hydrus +user=hydrus +directory=/opt/hydrus +command=python3 /opt/hydrus/client.py --db_dir %(ENV_DB_DIR)s %(ENV_HYDRUS_EXTRA)s +startretries=89 +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stderr_logfile=/dev/stderr +stdout_logfile_maxbytes=0 +stderr_logfile_maxbytes=0 diff --git a/static/build_files/docker/docker_build.yml b/static/build_files/docker/docker_build.yml new file mode 100644 index 00000000..424e670d --- /dev/null +++ b/static/build_files/docker/docker_build.yml @@ -0,0 +1,106 @@ +name: Build Containers +on: + push: + tags: + - 'v*' + workflow_dispatch: [] + +jobs: + build-client: + runs-on: [ubuntu-latest] + steps: + - + name: Checkout + uses: actions/checkout@v2.3.4 + - + name: Docker meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v2 + with: + images: | + ghcr.io/hydrusnetwork/hydrus + tags: | + type=edge + type=ref,event=pr + type=semver,pattern={{raw}} + labels: | + org.opencontainers.image.title=Hydrus Network + org.opencontainers.image.description=A personal booru-style media tagger that can import files and tags from your hard drive and popular websites. + org.opencontainers.image.vendor=hydrusnetwork + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + buildkitd-flags: "--debug" + - + name: Login to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + - + name: Build + uses: docker/build-push-action@v2 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + file: ./static/build_files/docker/client/Dockerfile + platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/386,linux/arm/v7 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + + build-server: + runs-on: [ubuntu-latest] + steps: + - + name: Checkout + uses: actions/checkout@v2.3.4 + - + name: Docker meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v2 + with: + images: | + ghcr.io/hydrusnetwork/hydrus + tags: | + type=edge + type=ref,event=pr + type=semver,pattern={{raw}} + flavor: | + latest=false + prefix=server- + labels: | + org.opencontainers.image.title=Hydrus Network Server + org.opencontainers.image.description=A personal booru-style media tagger that can import files and tags from your hard drive and popular websites. + org.opencontainers.image.vendor=hydrusnetwork + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + buildkitd-flags: "--debug" + - + name: Login to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + - + name: Build + uses: docker/build-push-action@v2 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + file: ./static/build_files/docker/server/Dockerfile + platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/386,linux/arm/v7 + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} \ No newline at end of file diff --git a/static/build_files/docker/server/Dockerfile b/static/build_files/docker/server/Dockerfile new file mode 100644 index 00000000..25424b43 --- /dev/null +++ b/static/build_files/docker/server/Dockerfile @@ -0,0 +1,29 @@ +FROM suika/opencv-video-minimal:4.2-py3.7.5 + +ARG UID +ARG GID + +RUN apk --no-cache add py3-beautifulsoup4 py3-psutil py3-pysocks py3-requests py3-twisted py3-yaml py3-lz4 ffmpeg py3-pillow py3-numpy py3-openssl py3-service_identity openssl su-exec +RUN pip install Send2Trash html5lib twisted cloudscrape + +RUN set -xe \ + && mkdir -p /opt/hydrus \ + && addgroup -g 1000 hydrus \ + && adduser -h /opt/hydrus -u 1000 -H -S -G hydrus hydrus + +COPY --chown=hydrus . /opt/hydrus +COPY --chown=hydrus --from=suika/swftools:2013-04-09-1007 /swftools/swfrender /opt/hydrus/bin/swfrender_linux + +VOLUME /opt/hydrus/db + +ENV UID=${UID:-1000} \ + GID=${GID:-1000} \ + MGMT_PORT=45870 + +EXPOSE ${MGMT_PORT} + +ENTRYPOINT ["/bin/sh", "/opt/hydrus/static/build_files/docker/server/entrypoint.sh"] + +HEALTHCHECK --interval=1m --timeout=10s --retries=3 --start-period=10s \ + CMD wget --quiet --tries=1 --no-check-certificate --spider \ + https://localhost:${MGMT_PORT} || exit 1 \ No newline at end of file diff --git a/static/build_files/docker/server/entrypoint.sh b/static/build_files/docker/server/entrypoint.sh new file mode 100644 index 00000000..cb9cbd79 --- /dev/null +++ b/static/build_files/docker/server/entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +USER_ID=${UID} +GROUP_ID=${GID} + +echo "Starting Hydrus with UID/GID : $USER_ID/$GROUP_ID" + +stop() { + python3 /opt/hydrus/server.py stop -d="/opt/hydrus/db" +} + +trap "stop" SIGTERM + +su-exec ${USER_ID}:${GROUP_ID} python3 /opt/hydrus/server.py -d="/opt/hydrus/db" --no_daemons & + +wait $!