RepoMirrors
/
hydrus
mirror of https://github.com/hydrusnetwork/hydrus.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Version 504

This commit is contained in:
Hydrus Network Developer 2022-10-26 15:43:00 -05:00
parent d718d15493
commit 08a3c34cb9
51 changed files with 3642 additions and 916 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

234
.github/workflows/release.yml vendored
View File

@ -5,63 +5,6 @@ on:
- 'v*'
jobs:
build-macos-Qt5:
runs-on: macos-11
steps:
-
name: Checkout
uses: actions/checkout@v3
-
name: Setup FFMPEG
uses: FedericoCarboni/setup-ffmpeg@v1.1.0
id: setup_ffmpeg
with:
token: ${{ secrets.GITHUB_TOKEN }}
-
name: Install mkdocs-material
run: python3 -m pip install mkdocs-material
-
name: Build docs to /help
run: mkdocs build -d help
-
name: Install PyOxidizer
run: python3 -m pip install pyoxidizer
-
name: Build Hydrus
run: |
cd $GITHUB_WORKSPACE
cp ${{ steps.setup_ffmpeg.outputs.ffmpeg-path }} bin/
cp static/build_files/macos/pyoxidizer.bzl pyoxidizer.bzl
cp static/build_files/macos/requirementsQt5.txt requirements.txt
basename $(rustc --print sysroot) | sed -e "s/^stable-//" > triple.txt
pyoxidizer build --release
cd build/$(head -n 1 triple.txt)/release
mkdir -p "Hydrus Network.app/Contents/MacOS"
mkdir -p "Hydrus Network.app/Contents/Resources"
mkdir -p "Hydrus Network.app/Contents/Frameworks"
mv install/static/icon.icns "Hydrus Network.app/Contents/Resources/icon.icns"
cp install/static/build_files/macos/Info.plist "Hydrus Network.app/Contents/Info.plist"
cp install/static/build_files/macos/ReadMeFirst.rtf ./ReadMeFirst.rtf
cp install/static/build_files/macos/running_from_app "install/running_from_app"
ln -s /Applications ./Applications
mv install/* "Hydrus Network.app/Contents/MacOS/"
rm -rf install
-
name: Build DMG
run: |
cd $GITHUB_WORKSPACE
temp_dmg="$(mktemp).dmg"
hdiutil create "$temp_dmg" -ov -volname "HydrusNetwork" -fs HFS+ -format UDZO -srcfolder "$GITHUB_WORKSPACE/build/$(head -n 1 triple.txt)/release"
mv "$temp_dmg" HydrusNetwork5.dmg
-
name: Upload a Build Artifact
uses: actions/upload-artifact@v3
with:
name: MacOS-DMG5
path: HydrusNetwork5.dmg
if-no-files-found: error
retention-days: 2
build-macos-Qt6:
runs-on: macos-11
steps:
@ -119,78 +62,6 @@ jobs:
if-no-files-found: error
retention-days: 2
build-ubuntu-Qt5:
runs-on: ubuntu-18.04
steps:
-
name: Checkout
uses: actions/checkout@v3
with:
path: hydrus
-
name: Setup Python
uses: actions/setup-python@v4
with:
python-version: 3.8
architecture: x64
-
name: APT Install
run: |
sudo apt-get update
sudo apt-get install -y libmpv1
-
name: Pip Install
run: python3 -m pip install -r hydrus/static/build_files/linux/requirementsQt5.txt
-
name: Build docs to /help
run: mkdocs build -d help
working-directory: hydrus
#- name: Cache Qt
# id: cache-qt
# uses: actions/cache@v1
# with:
# path: Qt
# key: ${{ runner.os }}-QtCache
#-
# name: Install Qt
# uses: jurplel/install-qt-action@v2
# with:
# install-deps: true
# setup-python: 'false'
# modules: qtcharts qtwidgets qtgui qtcore
# cached: ${{ steps.cache-qt.outputs.cache-hit }}
-
name: Build Hydrus
run: |
cp hydrus/static/build_files/linux/client.spec client.spec
cp hydrus/static/build_files/linux/server.spec server.spec
pyinstaller server.spec
pyinstaller client.spec
-
name: Remove Chonk
run: |
find dist/client/ -type f -name "*.pyc" -delete
while read line; do find dist/client/ -type f -name "${line}" -delete ; done < hydrus/static/build_files/linux/files_to_delete.txt
-
name: Set Permissions
run: |
sudo chown --recursive 1000:1000 dist/client
sudo find dist/client -type d -exec chmod 0755 {} \;
sudo chmod +x dist/client/client dist/client/server dist/client/bin/swfrender_linux
-
name: Compress Client
run: |
mv dist/client "dist/Hydrus Network"
tar -czvf Ubuntu-Extract5.tar.gz -C dist "Hydrus Network"
-
name: Upload a Build Artifact
uses: actions/upload-artifact@v3
with:
name: Ubuntu-Extract5
path: Ubuntu-Extract5.tar.gz
if-no-files-found: error
retention-days: 2
build-ubuntu-Qt6:
runs-on: ubuntu-20.04
steps:
@ -267,87 +138,6 @@ jobs:
if-no-files-found: error
retention-days: 2
build-windows-Qt5:
runs-on: windows-2019
steps:
-
name: Checkout
uses: actions/checkout@v3
with:
path: hydrus
-
name: Setup FFMPEG
uses: FedericoCarboni/setup-ffmpeg@v1.1.0
id: setup_ffmpeg
with:
token: ${{ secrets.GITHUB_TOKEN }}
-
name: Setup Python
uses: actions/setup-python@v4
with:
python-version: 3.8
architecture: x64
-
name: Pip Install
run: python3 -m pip install -r hydrus/static/build_files/windows/requirementsQt5.txt
-
name: Build docs to /help
run: mkdocs build -d help
working-directory: hydrus
#-
# name: Cache Qt
# id: cache_qt
# uses: actions/cache@v1
# with:
# path: ../Qt
# key: ${{ runner.os }}-QtCache
#-
# name: Install Qt
# uses: jurplel/install-qt-action@v2
# with:
# install-deps: true
# setup-python: 'false'
# modules: qtcharts qtwidgets qtgui qtcore
# cached: ${{ steps.cache_qt.outputs.cache-hit }}
-
name: Download mpv-dev
uses: carlosperate/download-file-action@v1.1.1
id: download_mpv
with:
file-url: 'https://sourceforge.net/projects/mpv-player-windows/files/libmpv/mpv-dev-x86_64-20210228-git-d1be8bb.7z'
file-name: 'mpv-dev-x86_64.7z'
location: '.'
-
name: Process mpv-dev
run: |
7z x ${{ steps.download_mpv.outputs.file-path }}
move mpv-1.dll hydrus\
-
name: Build Hydrus
run: |
move ${{ steps.setup_ffmpeg.outputs.ffmpeg-path }} hydrus\bin\
move hydrus\static\build_files\windows\sqlite3.dll hydrus\
move hydrus\static\build_files\windows\sqlite3.exe hydrus\db
move hydrus\static\build_files\windows\client-winQt5.spec client-win.spec
move hydrus\static\build_files\windows\server-win.spec server-win.spec
pyinstaller server-win.spec
pyinstaller client-win.spec
dir -r
-
name: Compress Client
run: |
cd .\dist
7z.exe a -tzip -mm=Deflate -mx=5 ..\Windows-Extract5.zip 'Hydrus Network'
cd ..
-
name: Upload a Build Artifact
uses: actions/upload-artifact@v3
with:
name: Windows-Extract5
path: Windows-Extract5.zip
if-no-files-found: error
retention-days: 2
build-windows-Qt6:
runs-on: windows-2019
steps:
@ -447,7 +237,7 @@ jobs:
create-release:
name: Create Release Entry
runs-on: ubuntu-20.04
needs: [build-windows-Qt5, build-windows-Qt6, build-ubuntu-Qt5, build-ubuntu-Qt6, build-macos-Qt5, build-macos-Qt6]
needs: [build-windows-Qt6, build-ubuntu-Qt6, build-macos-Qt6]
steps:
-
name: Checkout code
@ -465,25 +255,19 @@ jobs:
name: Rename Files
run: |
mkdir ubuntu windows
mv MacOS-DMG5/HydrusNetwork5.dmg Hydrus.Network.${{ env.version_short }}.-.macOS.Qt5.-.App.dmg
mv MacOS-DMG6/HydrusNetwork6.dmg Hydrus.Network.${{ env.version_short }}.-.macOS.Qt6.-.App.dmg
mv Windows-Install/HydrusInstaller.exe Hydrus.Network.${{ env.version_short }}.-.Windows.Qt6.-.Installer.exe
mv Windows-Extract5/Windows-Extract5.zip Hydrus.Network.${{ env.version_short }}.-.Windows.Qt5.-.Extract.only.zip
mv Windows-Extract6/Windows-Extract6.zip Hydrus.Network.${{ env.version_short }}.-.Windows.Qt6.-.Extract.only.zip
mv Ubuntu-Extract5/Ubuntu-Extract5.tar.gz Hydrus.Network.${{ env.version_short }}.-.Linux.Qt5.-.Executable.tar.gz
mv Ubuntu-Extract6/Ubuntu-Extract6.tar.gz Hydrus.Network.${{ env.version_short }}.-.Linux.Qt6.-.Executable.tar.gz
mv Windows-Install/HydrusInstaller.exe Hydrus.Network.${{ env.version_short }}.-.Windows.-.Installer.exe
mv Windows-Extract6/Windows-Extract6.zip Hydrus.Network.${{ env.version_short }}.-.Windows.-.Extract.only.zip
mv Ubuntu-Extract6/Ubuntu-Extract6.tar.gz Hydrus.Network.${{ env.version_short }}.-.Linux.-.Executable.tar.gz
mv MacOS-DMG6/HydrusNetwork6.dmg Hydrus.Network.${{ env.version_short }}.-.macOS.-.App.dmg
-
name: Release new
uses: softprops/action-gh-release@v1
if: startsWith(github.ref, 'refs/tags/')
with:
files: |
Hydrus.Network.${{ env.version_short }}.-.Windows.Qt6.-.Installer.exe
Hydrus.Network.${{ env.version_short }}.-.Windows.Qt5.-.Extract.only.zip
Hydrus.Network.${{ env.version_short }}.-.Windows.Qt6.-.Extract.only.zip
Hydrus.Network.${{ env.version_short }}.-.Linux.Qt5.-.Executable.tar.gz
Hydrus.Network.${{ env.version_short }}.-.Linux.Qt6.-.Executable.tar.gz
Hydrus.Network.${{ env.version_short }}.-.macOS.Qt5.-.App.dmg
Hydrus.Network.${{ env.version_short }}.-.macOS.Qt6.-.App.dmg
Hydrus.Network.${{ env.version_short }}.-.Windows.-.Installer.exe
Hydrus.Network.${{ env.version_short }}.-.Windows.-.Extract.only.zip
Hydrus.Network.${{ env.version_short }}.-.Linux.-.Executable.tar.gz
Hydrus.Network.${{ env.version_short }}.-.macOS.-.App.dmg
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

71
docs/changelog.md
View File

@ -7,6 +7,55 @@ title: Changelog
!!! note
This is the new changelog, only the most recent builds. For all versions, see the [old changelog](old_changelog.html).
## [Version 504](https://github.com/hydrusnetwork/hydrus/releases/tag/v504)
### Qt5
* as a reminder, I am no longer supporting Qt5 with the official builds. if you are on Windows 7 (and I have heard at least one version of Win 8.1), or a similarly old OS, you likely cannot run the official builds now. if this is you, please check the 'running from source' guide in the help, which will allow you to keep updating the program. this process is now easy in Windows and should be similarly easy on other platforms soon
### misc
* if you run from source in windows, the program _should_ now have its own taskbar group and use the correct hydrus icon. if you try and pin it to taskbar, it will revert to the 'python' icon, but you can give a shortcut to a batch file an icon and pin that to start
* unfortunately, I have to remove the 'deviant art tag search' downloader this week. they killed the old API we were using, and what remaining open date-paginated search results the site offers is obfuscated and tokenised (no permanent links), more than I could quickly unravel. other downloader creators are welcome to give it a go. if you have a subscription for a da tag search, it will likely complain on its next run. please pause it and try to capture the best artists from that search (until DA kill their free artist api, then who knows what will happen). the oauth/phone app menace marches on
* focus on the thumbnail panel is now preserved whenever it swaps out for another (like when you refresh the search)
* fixed an issue where cancelling service selection on database->c&r->repopulate truncated would create an empty modal message
* fixed a stupid typo in the recently changed server petition counting auto-fixing code
### importer/exporter sidecar expansion
* when you import or export files from/to disk, either manually or automatically, the option to pull or send tags to .txt files is now expanded:
* - you can now import or export URLs
* - you can now read or write .json files
* - you can now import from or export to multiple sidecars, and have multiple separate pipelines
* - you can now give sidecar files suffixes, for ".tags.txt" and similar
* - you can now filter and transform all the strings in this pipeline using the powerful String Processor just like in the parsing system
* this affects manual imports, manual exports, import folders, and export folders. instead of smart .txt checkboxes, there's now a button leading to some nested dialogs to customise your 'routers' and, in manual imports, a new page tab in the 'add tags before import' window
* this bones of this system was already working in the background when I introduced it earlier this year, but now all components are exposed
* new export folders now start with the same default metadata migration as set in the last manual file export dialog
* this system will expand in future. most important is to add a 'favourites' system so you can easily save/load your different setups. then adding more content types (e.g. ratings) and .xml. I'd also like to add purely internal file-to-itself datatype transformation (e.g. pulling url:(url) tags and converting them to actual known urls, and vice versa)
### importer/exporter sidecar expansion (boring stuff)
* split the importer/exporter objects into separate importers and exporters. existing router objects will update and split their internal objects safely
* all objects in this system can now describe themselves
* all import/export nodes now produce appropriate example texts for string processing and parsing UI test panels
* Filename Tagging Options objects no longer track neighbouring .txt file importing, and their UI removes it too. Import Folders will suck their old data on update and convert to metadata routers
* wrote a json sidecar importer that takes a parsing formula
* wrote a json sidecar exporter that takes a list of dictionary names to export to. it will edit an existing file
* wrote some ui panels to edit single file metadata migration routers
* wrote some ui panels to edit single file metadata migration importers
* wrote some ui panels to edit single file metadata migration exporters
* updated edit export folder panel to use the new UI. it was already using a full static version of the system behind the scenes; now this is exposed and editable
* updated the manual file export panel to use the new UI. it was using a half version of the system before--now the default options are updated to the new router object and you can create multiple exports
* updated import folders to use the new UI. the filename tagging options no longer handles .txt, it is now on a separate button on the import folder
* updated manual file imports to use the new UI. the 'add tags before import' window now has a 'sidecars' page tab, which lets you edit metadata routers. it updates a path preview list live with what it expects to parse
* a full suite of new unit tests now checks the router, the four import nodes, and the four export nodes thoroughly
* renamed ClientExportingMetadata to ClientMetadataMigration and moved to the metadata module. refactored the importers, exporters, and shared methods to their own files in the same module
* created a gui.metadata module for the new router and metadata import/export widgets and panels
* created a gui.exporting module for the existing export folder and manual export gui code
* reworked some of the core importer/exporter objects and inheritance in clientmetadatamigration
* updated the HDDImport object and creation pipeline to handle metadata routers (as piped from the new sidecars tab)
* when the hdd import or import folder is set to delete original files, now all defined sidecars are deleted along with the media file
* cleaned up a bunch of related metadata importer/exporter code
* cleaned import folder code
* cleaned hdd importer code
## [Version 503](https://github.com/hydrusnetwork/hydrus/releases/tag/v503)
### misc
@ -391,25 +440,3 @@ _almost all the changes this week are only important to server admins and janito
* fiddled with QPoint and QPointF conversions a little so I _think_ Qt5 and Qt6 is always talking about the same type
* updated build scripts and requirements.txts for the new situation
* updated the help a bit for the new situation
## [Version 493](https://github.com/hydrusnetwork/hydrus/releases/tag/v493)
### EXIF
* in the first step of 'official' EXIF support, the media viewer now has a 'cog' button on the top hover, enabled when looking at a jpeg, that will check the file for EXIF data. if found, it will throw it up on a simple new window that shows EXIF id, label, and value. this is a hacked-together prototype, not super user-friendly, but it works. let me know what you think, and please send me any files that have weird EXIF that doesn't parse right but you think should. I already discovered a file with a null character that wouldn't display in UI, that sort of thing
* GPS EXIF values are also parsed and extracted
* made it so you can double-click a row in this new window to copy an EXIF value to clipboard
* in the duplicate filter, if one or both files have exif data, this is now noted in the comparison statements, just like ICC profile! (issue #469)
* obvious future extensions here will be storing 'has exif' in the database and allowing its presence to be searchable and enabling the cog button (or a nicer 'exif' button) only when there is known data to see. a subsequent step would be actually caching the data in the database for full EXIF search
* as a side thing, we're now set up on the hydrus end to pull TIFF EXIF, but PIL doesn't seem to offer it, so we'll have to wait for a different solution there
### fixes and misc
* fixed a problem that made saved page file sorts reset their sort order one time on update to v492. thank you to a user for noticing this and discovering the fix, and I'm very sorry for the inconvenience of changing your session and favourite search sorts. unfortunately there is no easy fix other than rolling back to a backup and jumping forward to this version
* fixed a v492 message display error when setting various duplicate relationships to three or more thumbnails at once. it was a stupid typo, sorry for the trouble! (issue #1199)
* if a page tab name elides to a 'shorter...' length, it now has its full name as the tooltip
* fixed a typo in update code error handling (issue #1192)
* the duplicate filter page now remembers if you are 'searching immediately'/'search paused' (issue #1193)
* if you are on non-Windows and export files manually or with an export folder to an NTFS or exFAT partition, this is now detected, and NTFS-invalid characters in the pattern-generated folders or filename are now replaced with underscores (issue #1194)
* 'fixed' a system predicate bug in the 'OR*' advanced predicate parser--entering a logical expression that results in a negated system tag now causes an error. previously, it would strip the 'system:' and just enter the given text as an unnamespaced tag. furthermore, that dialog now reports specific error reasons when it fails to parse. I hope to improve support for negated system tags in future--some stuff, like archive/inbox, should be easy.
* I think I fixed an instance where the archive/delete filter's confirmation dialog could present 'delete from hard disk' as an option when it wasn't appropriate
* in an attempt to reduce the media-change flickering we've recently seen in the media viewer, I untangled a bunch of the canvas size/position code this week. I'm preparing a complete overhaul and neat Qt layout integration, which this starts. I _think_ I've made some things less flickery on occasion, but we'll see IRL. much more to do
* added a '--profile_mode' launch argument, which allows you to capture the performance of boot and also try out profile mode on the server (although support there is very limited atm)

49
docs/old_changelog.html
View File

@ -33,6 +33,55 @@
<div class="content">
<h3 id="changelog"><a href="#changelog">changelog</a></h3>
<ul>
<li><h3 id="version_504"><a href="#version_504">version 504</a></h3></li>
<ul>
<li>Qt5:</li>
<li>as a reminder, I am no longer supporting Qt5 with the official builds. if you are on Windows 7 (and I have heard at least one version of Win 8.1), or a similarly old OS, you likely cannot run the official builds now. if this is you, please check the 'running from source' guide in the help, which will allow you to keep updating the program. this process is now easy in Windows and should be similarly easy on other platforms soon</li>
<li>.</li>
<li>misc:</li>
<li>if you run from source in windows, the program _should_ now have its own taskbar group and use the correct hydrus icon. if you try and pin it to taskbar, it will revert to the 'python' icon, but you can give a shortcut to a batch file an icon and pin that to start</li>
<li>unfortunately, I have to remove the 'deviant art tag search' downloader this week. they killed the old API we were using, and what remaining open date-paginated search results the site offers is obfuscated and tokenised (no permanent links), more than I could quickly unravel. other downloader creators are welcome to give it a go. if you have a subscription for a da tag search, it will likely complain on its next run. please pause it and try to capture the best artists from that search (until DA kill their free artist api, then who knows what will happen). the oauth/phone app menace marches on</li>
<li>focus on the thumbnail panel is now preserved whenever it swaps out for another (like when you refresh the search)</li>
<li>fixed an issue where cancelling service selection on database->c&r->repopulate truncated would create an empty modal message</li>
<li>fixed a stupid typo in the recently changed server petition counting auto-fixing code</li>
<li>.</li>
<li>importer/exporter sidecar expansion:</li>
<li>when you import or export files from/to disk, either manually or automatically, the option to pull or send tags to .txt files is now expanded:</li>
<li>- you can now import or export URLs</li>
<li>- you can now read or write .json files</li>
<li>- you can now import from or export to multiple sidecars, and have multiple separate pipelines</li>
<li>- you can now give sidecar files suffixes, for ".tags.txt" and similar</li>
<li>- you can now filter and transform all the strings in this pipeline using the powerful String Processor just like in the parsing system</li>
<li>this affects manual imports, manual exports, import folders, and export folders. instead of smart .txt checkboxes, there's now a button leading to some nested dialogs to customise your 'routers' and, in manual imports, a new page tab in the 'add tags before import' window</li>
<li>this bones of this system was already working in the background when I introduced it earlier this year, but now all components are exposed</li>
<li>new export folders now start with the same default metadata migration as set in the last manual file export dialog</li>
<li>this system will expand in future. most important is to add a 'favourites' system so you can easily save/load your different setups. then adding more content types (e.g. ratings) and .xml. I'd also like to add purely internal file-to-itself datatype transformation (e.g. pulling url:(url) tags and converting them to actual known urls, and vice versa)</li>
<li>.</li>
<li>importer/exporter sidecar expansion (boring stuff):</li>
<li>split the importer/exporter objects into separate importers and exporters. existing router objects will update and split their internal objects safely</li>
<li>all objects in this system can now describe themselves</li>
<li>all import/export nodes now produce appropriate example texts for string processing and parsing UI test panels</li>
<li>Filename Tagging Options objects no longer track neighbouring .txt file importing, and their UI removes it too. Import Folders will suck their old data on update and convert to metadata routers</li>
<li>wrote a json sidecar importer that takes a parsing formula</li>
<li>wrote a json sidecar exporter that takes a list of dictionary names to export to. it will edit an existing file</li>
<li>wrote some ui panels to edit single file metadata migration routers</li>
<li>wrote some ui panels to edit single file metadata migration importers</li>
<li>wrote some ui panels to edit single file metadata migration exporters</li>
<li>updated edit export folder panel to use the new UI. it was already using a full static version of the system behind the scenes; now this is exposed and editable</li>
<li>updated the manual file export panel to use the new UI. it was using a half version of the system before--now the default options are updated to the new router object and you can create multiple exports</li>
<li>updated import folders to use the new UI. the filename tagging options no longer handles .txt, it is now on a separate button on the import folder</li>
<li>updated manual file imports to use the new UI. the 'add tags before import' window now has a 'sidecars' page tab, which lets you edit metadata routers. it updates a path preview list live with what it expects to parse</li>
<li>a full suite of new unit tests now checks the router, the four import nodes, and the four export nodes thoroughly</li>
<li>renamed ClientExportingMetadata to ClientMetadataMigration and moved to the metadata module. refactored the importers, exporters, and shared methods to their own files in the same module</li>
<li>created a gui.metadata module for the new router and metadata import/export widgets and panels</li>
<li>created a gui.exporting module for the existing export folder and manual export gui code</li>
<li>reworked some of the core importer/exporter objects and inheritance in clientmetadatamigration</li>
<li>updated the HDDImport object and creation pipeline to handle metadata routers (as piped from the new sidecars tab)</li>
<li>when the hdd import or import folder is set to delete original files, now all defined sidecars are deleted along with the media file</li>
<li>cleaned up a bunch of related metadata importer/exporter code</li>
<li>cleaned import folder code</li>
<li>cleaned hdd importer code</li>
</ul>
<li><h3 id="version_503"><a href="#version_503">version 503</a></h3></li>
<ul>
<li>misc:</li>

4
docs/running_from_source.md
View File

@ -61,7 +61,7 @@ There are three external libraries. You just have to get them and put them in th
Just double-click the batch file, and it will take you through the setup. It should take a minute to download and a couple minutes to install. If it seems like it hung, just give it time to finish. It'll say 'Done!' when it is done.
If something messes up, or you want to switch between Qt5/Qt6, just run the batch again and you will have an option to reinstall everything. Everything these scripts do ends up in the 'venv' directory, so you can also just delete that folder to 'uninstall'. It should just 'work' on most normal computers, but let me know if you have any trouble.
If something messes up, or you want to switch between Qt5/Qt6, just run the batch again and you will have an option to reinstall everything. Everything these scripts do ends up in the 'venv' directory, so you can also just delete that folder to 'uninstall'. It should 'just work' on most normal computers, but let me know if you have any trouble.
Then run 'setup_help.bat' to build the help. This isn't necessary, but it is nice to have it built locally. You can run this again at any time to rebuild the current help.
@ -71,6 +71,8 @@ Then run 'client.bat' to start the client. The first start will take a little lo
If you want to redirect your database or use any other launch arguments, then copy 'client.bat' to 'client-user.bat' and edit it, inserting your desired db path. Run this instead of 'client.bat'. New `git pull` commands will not affect 'client-user.bat'.
You probably can't pin your .bat file to your Taskbar or Start (and if you try and pin the running program to your taskbar, its icon may revert to Python), but you can make a shortcut to the .bat file, pin that to Start, and in its properties set a custom icon. There's a nice hydrus one in `install_dir/static`.
## Simple Windows Updating Guide
To update, you do the same thing as for the extract builds.

17
hydrus/client/ClientController.py
View File

@ -1595,6 +1595,23 @@ class Controller( HydrusController.HydrusController ):
ClientGUICore.GUICore()
if HC.PLATFORM_WINDOWS:
try:
# this makes the 'application user model' of the program unique, allowing instantiations to group on their own taskbar icon
# also allows the window icon to go to the taskbar icon, instead of python if you are running from source
import ctypes
ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID( 'hydrus network client' )
except:
pass
self.app = App( self._pubsub, sys.argv )
self.main_qt_thread = self.app.thread()

57
hydrus/client/ClientOptions.py
View File

@ -18,7 +18,7 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_CLIENT_OPTIONS
SERIALISABLE_NAME = 'Client Options'
SERIALISABLE_VERSION = 4
SERIALISABLE_VERSION = 5
def __init__( self ):
@ -454,8 +454,6 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
self._dictionary[ 'key_list' ] = {}
self._dictionary[ 'key_list' ][ 'default_neighbouring_txt_tag_service_keys' ] = []
#
self._dictionary[ 'noneable_integers' ] = {}
@ -716,6 +714,10 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
self._dictionary[ 'default_tag_sort' ] = ClientTagSorting.TagSort.STATICGetTextASCDefault()
#
self._dictionary[ 'default_export_files_metadata_routers' ] = HydrusSerialisable.SerialisableList()
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
@ -883,6 +885,39 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
return ( 4, new_serialisable_info )
if version == 4:
serialisable_dictionary = old_serialisable_info
loaded_dictionary = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_dictionary )
if 'key_list' in loaded_dictionary and 'default_neighbouring_txt_tag_service_keys' in loaded_dictionary[ 'key_list' ]:
encoded_default_neighbouring_txt_tag_service_keys = loaded_dictionary[ 'key_list' ][ 'default_neighbouring_txt_tag_service_keys' ]
default_neighbouring_txt_tag_service_keys = [ bytes.fromhex( hex_key ) for hex_key in encoded_default_neighbouring_txt_tag_service_keys ]
from hydrus.client.metadata import ClientMetadataMigration
from hydrus.client.metadata import ClientMetadataMigrationExporters
from hydrus.client.metadata import ClientMetadataMigrationImporters
importers = [ ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaTags( service_key = service_key ) for service_key in default_neighbouring_txt_tag_service_keys ]
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT()
metadata_router = ClientMetadataMigration.SingleFileMetadataRouter( importers = importers, exporter = exporter )
metadata_routers = [ metadata_router ]
loaded_dictionary[ 'default_export_files_metadata_routers' ] = HydrusSerialisable.SerialisableList( metadata_routers )
del loaded_dictionary[ 'key_list' ][ 'default_neighbouring_txt_tag_service_keys' ]
new_serialisable_info = loaded_dictionary.GetSerialisableTuple()
return ( 5, new_serialisable_info )
def ClearCustomDefaultSystemPredicates( self, predicate_type = None, comparable_predicate = None ):
@ -969,6 +1004,14 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
def GetDefaultExportFilesMetadataRouters( self ):
with self._lock:
return list( self._dictionary[ 'default_export_files_metadata_routers' ] )
def GetDefaultFileImportOptions( self, options_type ):
with self._lock:
@ -1442,6 +1485,14 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
def SetDefaultExportFilesMetadataRouters( self, metadata_routers ):
with self._lock:
self._dictionary[ 'default_export_files_metadata_routers' ] = HydrusSerialisable.SerialisableList( metadata_routers )
def SetDefaultFileImportOptions( self, options_type, file_import_options ):
with self._lock:

6
hydrus/client/ClientStrings.py
View File

@ -9,7 +9,6 @@ import urllib.parse
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusGlobals as HG
from hydrus.core import HydrusSerialisable
from hydrus.core import HydrusTags
@ -1242,6 +1241,11 @@ class StringProcessor( StringProcessingStep ):
return proc_strings
def MakesChanges( self ) -> bool:
return True in ( step.MakesChanges() for step in self._processing_steps )
def ProcessStrings( self, starting_strings: typing.Iterable[ str ], max_steps_allowed = None, no_slicing = False ) -> typing.List[ str ]:
current_strings = list( starting_strings )

34
hydrus/client/db/ClientDB.py
View File

@ -10555,6 +10555,40 @@ class DB( HydrusDB.HydrusDB ):
if version == 503:
try:
domain_manager = self.modules_serialisable.GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER )
domain_manager.Initialise()
#
# no longer supported, they nuked the open api
domain_manager.DeleteGUGs( (
'deviant art tag search',
) )
#
domain_manager.TryToLinkURLClassesAndParsers()
#
self.modules_serialisable.SetJSONDump( domain_manager )
except Exception as e:
HydrusData.PrintException( e )
message = 'Trying to update some downloader objects failed! Please let hydrus dev know!'
self.pub_initial_message( message )
self._controller.frame_splash_status.SetTitleText( 'updated db to v{}'.format( HydrusData.ToHumanInt( version + 1 ) ) )
self._Execute( 'UPDATE version SET version = ?;', ( version + 1, ) )

5
hydrus/client/db/ClientDBFilesStorage.py
View File

@ -218,6 +218,11 @@ class DBLocationContextBranch( DBLocationContext, ClientDBModule.ClientDBModule
return '{} CROSS JOIN {} USING ( hash_id )'.format( table_phrase, self.SINGLE_TABLE_NAME )
def GetTablesAndColumnsThatUseDefinitions( self, content_type: int ) -> typing.List[ typing.Tuple[ str, str ] ]:
return []
def SingleTableIsFast( self ) -> bool:
return False

2
hydrus/client/db/ClientDBMaintenance.py
View File

@ -282,7 +282,7 @@ class ClientDBMaintenance( ClientDBModule.ClientDBModule ):
def RegisterShutdownWork( self ):
self._Execute( 'DELETE from last_shutdown_work_time;' )
self._Execute( 'DELETE FROM last_shutdown_work_time;' )
self._Execute( 'INSERT INTO last_shutdown_work_time ( last_shutdown_work_time ) VALUES ( ? );', ( HydrusData.GetNow(), ) )

5
hydrus/client/db/ClientDBModule.py
View File

@ -68,3 +68,8 @@ class ClientDBModule( HydrusDBModule.HydrusDBModule ):
HG.client_controller.frame_splash_status.SetText( 'recreating tables' )
def GetTablesAndColumnsThatUseDefinitions( self, content_type: int ) -> typing.List[ typing.Tuple[ str, str ] ]:
raise NotImplementedError()

6
hydrus/client/exporting/ClientExportingFiles.py
View File

@ -15,10 +15,8 @@ from hydrus.core import HydrusThreading
from hydrus.client import ClientConstants as CC
from hydrus.client import ClientPaths
from hydrus.client import ClientSearch
from hydrus.client.exporting import ClientExportingMetadata
from hydrus.client.media import ClientMediaManagers
from hydrus.client.metadata import ClientMetadataMigration
from hydrus.client.metadata import ClientTags
from hydrus.client.metadata import ClientTagSorting
MAX_PATH_LENGTH = 240 # bit of padding from 255 for .txt neigbouring and other surprises
@ -663,7 +661,7 @@ class ExportFolder( HydrusSerialisable.SerialisableBaseNamed ):
return self._last_error
def GetMetadataRouters( self ) -> typing.Collection[ ClientExportingMetadata.SingleFileMetadataRouter ]:
def GetMetadataRouters( self ) -> typing.Collection[ ClientMetadataMigration.SingleFileMetadataRouter ]:
return self._metadata_routers

321
hydrus/client/exporting/ClientExportingMetadata.py
View File

@ -1,321 +0,0 @@
import os
import typing
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusGlobals as HG
from hydrus.core import HydrusSerialisable
from hydrus.core import HydrusTags
from hydrus.core import HydrusText
from hydrus.client import ClientStrings
from hydrus.client.media import ClientMediaResult
from hydrus.client.metadata import ClientTags
def GetSidecarPath( actual_file_path: str, suffix: str, file_extension: str ):
path_components = [ actual_file_path ]
if suffix != '':
path_components.append( suffix )
path_components.append( file_extension )
return '.'.join( path_components )
class SingleFileMetadataExporterMedia( object ):
def Export( self, hash: bytes, rows: typing.Collection[ str ] ):
raise NotImplementedError()
class SingleFileMetadataImporterMedia( object ):
def Import( self, media_result: ClientMediaResult.MediaResult ):
raise NotImplementedError()
class SingleFileMetadataExporterSidecar( object ):
def Export( self, actual_file_path: str, rows: typing.Collection[ str ] ):
raise NotImplementedError()
class SingleFileMetadataImporterSidecar( object ):
def Import( self, actual_file_path: str ):
raise NotImplementedError()
# TODO: add ToString and any other stuff here so this can all show itself prettily in a listbox
# 'I grab a .reversotags.txt sidecar and reverse the text and then send it as tags to my tags'
class SingleFileMetadataRouter( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_ROUTER
SERIALISABLE_NAME = 'Metadata Single File Converter'
SERIALISABLE_VERSION = 1
def __init__( self, importers = None, string_processor = None, exporter = None ):
if importers is None:
importers = []
if string_processor is None:
string_processor = ClientStrings.StringProcessor()
if exporter is None:
exporter = SingleFileMetadataImporterExporterTXT()
HydrusSerialisable.SerialisableBase.__init__( self )
self._importers = HydrusSerialisable.SerialisableList( importers )
self._string_processor = string_processor
self._exporter = exporter
def _GetSerialisableInfo( self ):
serialisable_importers = self._importers.GetSerialisableTuple()
serialisable_string_processor = self._string_processor.GetSerialisableTuple()
serialisable_exporter = self._exporter.GetSerialisableTuple()
return ( serialisable_importers, serialisable_string_processor, serialisable_exporter )
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_importers, serialisable_string_processor, serialisable_exporter ) = serialisable_info
self._importers = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_importers )
self._string_processor = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_string_processor )
self._exporter = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_exporter )
def GetExporter( self ):
return self._exporter
def GetImportedSidecarTexts( self, file_path: str, and_process_them = True ):
rows = set()
for importer in self._importers:
if isinstance( importer, SingleFileMetadataImporterSidecar ):
rows.update( importer.Import( file_path ) )
else:
raise Exception( 'This convertor does not import from a sidecar!' )
rows = sorted( rows, key = HydrusTags.ConvertTagToSortable )
if and_process_them:
rows = self._string_processor.ProcessStrings( starting_strings = rows )
return rows
def GetImporters( self ):
return self._importers
def Work( self, media_result: ClientMediaResult.MediaResult, file_path: str ):
rows = set()
for importer in self._importers:
if isinstance( importer, SingleFileMetadataImporterSidecar ):
rows.update( importer.Import( file_path ) )
elif isinstance( importer, SingleFileMetadataImporterMedia ):
rows.update( importer.Import( media_result ) )
else:
raise Exception( 'Problem with importer object!' )
rows = sorted( rows, key = HydrusTags.ConvertTagToSortable )
rows = self._string_processor.ProcessStrings( starting_strings = rows )
if len( rows ) == 0:
return
if isinstance( self._exporter, SingleFileMetadataExporterSidecar ):
self._exporter.Export( file_path, rows )
elif isinstance( self._exporter, SingleFileMetadataExporterMedia ):
self._exporter.Export( media_result.GetHash(), rows )
else:
raise Exception( 'Problem with exporter object!' )
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_ROUTER ] = SingleFileMetadataRouter
class SingleFileMetadataImporterExporterMediaTags( HydrusSerialisable.SerialisableBase, SingleFileMetadataExporterMedia, SingleFileMetadataImporterMedia ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_EXPORTER_MEDIA_TAGS
SERIALISABLE_NAME = 'Metadata Single File Importer Exporter Media Tags'
SERIALISABLE_VERSION = 1
def __init__( self, service_key = None ):
HydrusSerialisable.SerialisableBase.__init__( self )
SingleFileMetadataExporterMedia.__init__( self )
SingleFileMetadataImporterMedia.__init__( self )
self._service_key = service_key
def _GetSerialisableInfo( self ):
return self._service_key.hex()
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
serialisable_service_key = serialisable_info
self._service_key = bytes.fromhex( serialisable_service_key )
def GetServiceKey( self ) -> bytes:
return self._service_key
def Import( self, media_result: ClientMediaResult.MediaResult ):
tags = media_result.GetTagsManager().GetCurrent( self._service_key, ClientTags.TAG_DISPLAY_STORAGE )
return tags
def Export( self, hash: bytes, rows: typing.Collection[ str ] ):
if HG.client_controller.services_manager.GetServiceType( self._service_key ) == HC.LOCAL_TAG:
add_content_action = HC.CONTENT_UPDATE_ADD
else:
add_content_action = HC.CONTENT_UPDATE_PEND
hashes = { hash }
content_updates = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, add_content_action, ( tag, hashes ) ) for tag in rows ]
HG.client_controller.WriteSynchronous( 'content_updates', { self._service_key : content_updates } )
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_EXPORTER_MEDIA_TAGS ] = SingleFileMetadataImporterExporterMediaTags
class SingleFileMetadataImporterExporterTXT( HydrusSerialisable.SerialisableBase, SingleFileMetadataExporterSidecar, SingleFileMetadataImporterSidecar ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_EXPORTER_TXT
SERIALISABLE_NAME = 'Metadata Single File Importer Exporter TXT'
SERIALISABLE_VERSION = 1
def __init__( self, suffix = None ):
HydrusSerialisable.SerialisableBase.__init__( self )
SingleFileMetadataExporterSidecar.__init__( self )
SingleFileMetadataImporterSidecar.__init__( self )
if suffix is None:
suffix = ''
self._suffix = suffix
def _GetSerialisableInfo( self ):
return self._suffix
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
self._suffix = serialisable_info
def Export( self, actual_file_path: str, rows: typing.Collection[ str ] ):
path = GetSidecarPath( actual_file_path, self._suffix, 'txt' )
with open( path, 'w', encoding = 'utf-8' ) as f:
f.write( '\n'.join( rows ) )
def Import( self, actual_file_path: str ) -> typing.Collection[ str ]:
path = GetSidecarPath( actual_file_path, self._suffix, 'txt' )
if not os.path.exists( path ):
return []
try:
with open( path, 'r', encoding = 'utf-8' ) as f:
raw_text = f.read()
except Exception as e:
raise Exception( 'Could not import from {}: {}'.format( path, str( e ) ) )
rows = HydrusText.DeserialiseNewlinedTexts( raw_text )
return rows
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_EXPORTER_TXT ] = SingleFileMetadataImporterExporterTXT

10
hydrus/client/gui/ClientGUI.py
View File

@ -53,7 +53,6 @@ from hydrus.client.gui import ClientGUIDialogsManage
from hydrus.client.gui import ClientGUIDialogsQuick
from hydrus.client.gui import ClientGUIDownloaders
from hydrus.client.gui import ClientGUIDragDrop
from hydrus.client.gui import ClientGUIExport
from hydrus.client.gui import ClientGUIFrames
from hydrus.client.gui import ClientGUIFunctions
from hydrus.client.gui import ClientGUILogin
@ -79,6 +78,7 @@ from hydrus.client.gui import ClientGUILocatorSearchProviders
from hydrus.client.gui import QtInit
from hydrus.client.gui import QtPorting as QP
from hydrus.client.gui.canvas import ClientGUIMPV
from hydrus.client.gui.exporting import ClientGUIExport
from hydrus.client.gui.importing import ClientGUIImport
from hydrus.client.gui.importing import ClientGUIImportFolders
from hydrus.client.gui.importing import ClientGUIImportOptions
@ -5324,8 +5324,6 @@ class FrameGUI( ClientGUITopLevelWindows.MainFrameThatResizes, CAC.ApplicationCo
job_key.SetVariable( 'popup_text_title', 'repopulating mapping tables' )
self._controller.pub( 'modal_message', job_key )
try:
tag_service_key = GetTagServiceKeyForMaintenance( self )
@ -5335,6 +5333,8 @@ class FrameGUI( ClientGUITopLevelWindows.MainFrameThatResizes, CAC.ApplicationCo
return
self._controller.pub( 'modal_message', job_key )
self._controller.Write( 'repopulate_mappings_from_cache', tag_service_key = tag_service_key, job_key = job_key )
@ -7375,9 +7375,9 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
self._menu_updater_database.update()
def NewPageImportHDD( self, paths, file_import_options, paths_to_additional_service_keys_to_tags, delete_after_success ):
def NewPageImportHDD( self, paths, file_import_options, metadata_routers, paths_to_additional_service_keys_to_tags, delete_after_success ):
management_controller = ClientGUIManagement.CreateManagementControllerImportHDD( paths, file_import_options, paths_to_additional_service_keys_to_tags, delete_after_success )
management_controller = ClientGUIManagement.CreateManagementControllerImportHDD( paths, file_import_options, metadata_routers, paths_to_additional_service_keys_to_tags, delete_after_success )
self._notebook.NewPage( management_controller, on_deepest_notebook = True )

2
hydrus/client/gui/ClientGUIDialogsQuick.py
View File

@ -214,6 +214,8 @@ def SelectServiceKey( service_types = HC.ALL_SERVICES, service_keys = None, unal
service_keys = [ service.GetServiceKey() for service in services ]
service_keys = set( service_keys )
if unallowed is not None:
service_keys.difference_update( unallowed )

13
hydrus/client/gui/ClientGUIScrolledPanelsReview.py
View File

@ -3136,7 +3136,7 @@ class ReviewLocalFileImports( ClientGUIScrolledPanels.ReviewPanel ):
self._add_button = ClientGUICommon.BetterButton( self, 'import now', self._DoImport )
self._add_button.setObjectName( 'HydrusAccept' )
self._tag_button = ClientGUICommon.BetterButton( self, 'add tags before the import >>', self._AddTags )
self._tag_button = ClientGUICommon.BetterButton( self, 'add tags/urls with the import >>', self._AddTags )
self._tag_button.setObjectName( 'HydrusAccept' )
self._tag_button.setToolTip( 'You can add specific tags to these files, import from sidecar files, or generate them based on filename. Don\'t be afraid to experiment!' )
@ -3213,6 +3213,10 @@ class ReviewLocalFileImports( ClientGUIScrolledPanels.ReviewPanel ):
def _AddTags( self ):
# TODO: convert this class to have a filenametaggingoptions and the structure for 'tags for these files', which is separate
# then make this button not start the import. just edit the options and routers and return
# if needed, we convert to paths_to_additional_tags on ultimate ok, or we convert the hdd import to just hold service_keys_to_filenametaggingoptions, like an import folder does
paths = self._paths_list.GetData()
if len( paths ) > 0:
@ -3227,11 +3231,11 @@ class ReviewLocalFileImports( ClientGUIScrolledPanels.ReviewPanel ):
if dlg.exec() == QW.QDialog.Accepted:
paths_to_additional_service_keys_to_tags = panel.GetValue()
( metadata_routers, paths_to_additional_service_keys_to_tags ) = panel.GetValue()
delete_after_success = self._delete_after_success.isChecked()
HG.client_controller.pub( 'new_hdd_import', paths, file_import_options, paths_to_additional_service_keys_to_tags, delete_after_success )
HG.client_controller.pub( 'new_hdd_import', paths, file_import_options, metadata_routers, paths_to_additional_service_keys_to_tags, delete_after_success )
self._OKParent()
@ -3261,11 +3265,12 @@ class ReviewLocalFileImports( ClientGUIScrolledPanels.ReviewPanel ):
file_import_options = self._import_options_button.GetFileImportOptions()
metadata_routers = []
paths_to_additional_service_keys_to_tags = collections.defaultdict( ClientTags.ServiceKeysToTags )
delete_after_success = self._delete_after_success.isChecked()
HG.client_controller.pub( 'new_hdd_import', paths, file_import_options, paths_to_additional_service_keys_to_tags, delete_after_success )
HG.client_controller.pub( 'new_hdd_import', paths, file_import_options, metadata_routers, paths_to_additional_service_keys_to_tags, delete_after_success )
self._OKParent()

2
hydrus/client/gui/ClientGUITime.py
View File

@ -329,6 +329,8 @@ class TimeDeltaButton( QW.QPushButton ):
self._RefreshLabel()
class TimeDeltaCtrl( QW.QWidget ):
timeDeltaChanged = QC.Signal()

235
hydrus/client/gui/ClientGUIExport.py → hydrus/client/gui/exporting/ClientGUIExport.py
View File

@ -17,7 +17,6 @@ from hydrus.client import ClientLocation
from hydrus.client import ClientSearch
from hydrus.client import ClientThreading
from hydrus.client.exporting import ClientExportingFiles
from hydrus.client.exporting import ClientExportingMetadata
from hydrus.client.gui import ClientGUIDialogsQuick
from hydrus.client.gui import ClientGUIFunctions
from hydrus.client.gui import ClientGUIScrolledPanels
@ -27,9 +26,12 @@ from hydrus.client.gui import QtPorting as QP
from hydrus.client.gui.lists import ClientGUIListBoxes
from hydrus.client.gui.lists import ClientGUIListConstants as CGLC
from hydrus.client.gui.lists import ClientGUIListCtrl
from hydrus.client.gui.metadata import ClientGUIMetadataMigration
from hydrus.client.gui.search import ClientGUIACDropdown
from hydrus.client.gui.widgets import ClientGUICommon
from hydrus.client.media import ClientMedia
from hydrus.client.metadata import ClientMetadataMigrationExporters
from hydrus.client.metadata import ClientMetadataMigrationImporters
from hydrus.client.metadata import ClientTags
class EditExportFoldersPanel( ClientGUIScrolledPanels.EditPanel ):
@ -79,9 +81,20 @@ class EditExportFoldersPanel( ClientGUIScrolledPanels.EditPanel ):
file_search_context = ClientSearch.FileSearchContext( location_context = default_location_context )
metadata_routers = new_options.GetDefaultExportFilesMetadataRouters()
period = 15 * 60
export_folder = ClientExportingFiles.ExportFolder( name, path, export_type = export_type, delete_from_client_after_export = delete_from_client_after_export, file_search_context = file_search_context, period = period, phrase = phrase )
export_folder = ClientExportingFiles.ExportFolder(
name,
path,
export_type = export_type,
delete_from_client_after_export = delete_from_client_after_export,
file_search_context = file_search_context,
metadata_routers = metadata_routers,
period = period,
phrase = phrase
)
with ClientGUITopLevelWindowsPanels.DialogEdit( self, 'edit export folder' ) as dlg:
@ -258,13 +271,13 @@ class EditExportFolderPanel( ClientGUIScrolledPanels.EditPanel ):
#
self._metadata_routers_box = ClientGUICommon.StaticBox( self, 'metadata export' )
self._metadata_routers_box = ClientGUICommon.StaticBox( self, 'sidecar exporting' )
self._current_metadata_routers = list( export_folder.GetMetadataRouters() )
metadata_routers = export_folder.GetMetadataRouters()
allowed_importer_classes = [ ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaTags, ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaURLs ]
allowed_exporter_classes = [ ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT, ClientMetadataMigrationExporters.SingleFileMetadataExporterJSON ]
text = 'This will export all the files\' tags, newline separated, into .txts beside the files themselves.'
self._export_tag_txts_services_button = ClientGUICommon.BetterButton( self._metadata_routers_box, 'set tag .txt services', self._SetTxtServices )
self._metadata_routers_button = ClientGUIMetadataMigration.SingleFileMetadataRoutersButton( self._metadata_routers_box, metadata_routers, allowed_importer_classes, allowed_exporter_classes )
#
@ -340,7 +353,7 @@ If you select synchronise, be careful!'''
self._phrase_box.Add( phrase_hbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
self._metadata_routers_box.Add( self._export_tag_txts_services_button, CC.FLAGS_ON_RIGHT )
self._metadata_routers_box.Add( self._metadata_routers_button, CC.FLAGS_EXPAND_PERPENDICULAR )
vbox = QP.VBoxLayout()
@ -355,84 +368,10 @@ If you select synchronise, be careful!'''
self._UpdateTypeDeleteUI()
self._UpdateTxtButton()
self._type.currentIndexChanged.connect( self._UpdateTypeDeleteUI )
self._delete_from_client_after_export.clicked.connect( self.EventDeleteFilesAfterExport )
def _GetCurrentTxtTagServiceKeys( self ):
current_txt_tag_service_keys = set()
if len( self._current_metadata_routers ) > 0:
metadata_router = self._current_metadata_routers[0]
for importer in metadata_router.GetImporters():
if isinstance( importer, ClientExportingMetadata.SingleFileMetadataImporterExporterMediaTags ):
service_key = importer.GetServiceKey()
current_txt_tag_service_keys.add( service_key )
return current_txt_tag_service_keys
def _SetTxtServices( self ):
# TODO: obviously replace all this, and elsewhere, with a unified metadata router edit UI panel/button
current_txt_tag_service_keys = self._GetCurrentTxtTagServiceKeys()
services_manager = HG.client_controller.services_manager
tag_services = services_manager.GetServices( HC.REAL_TAG_SERVICES )
choice_tuples = [ ( service.GetName(), service.GetServiceKey(), service.GetServiceKey() in current_txt_tag_service_keys ) for service in tag_services ]
try:
neighbouring_txt_tag_service_keys = ClientGUIDialogsQuick.SelectMultipleFromList( self, 'select tag services', choice_tuples )
except HydrusExceptions.CancelledException:
return
importers = [ ClientExportingMetadata.SingleFileMetadataImporterExporterMediaTags( service_key ) for service_key in neighbouring_txt_tag_service_keys ]
exporter = ClientExportingMetadata.SingleFileMetadataImporterExporterTXT()
metadata_router = ClientExportingMetadata.SingleFileMetadataRouter( importers = importers, exporter = exporter )
self._current_metadata_routers = [ metadata_router ]
self._UpdateTxtButton()
def _UpdateTxtButton( self ):
current_txt_tag_service_keys = self._GetCurrentTxtTagServiceKeys()
if len( current_txt_tag_service_keys ) == 0:
tt = 'No services set.'
else:
names = sorted( [ HG.client_controller.services_manager.GetName( service_key ) for service_key in current_txt_tag_service_keys ] )
tt = ', '.join( names )
self._export_tag_txts_services_button.setToolTip( tt )
def _UpdateTypeDeleteUI( self ):
if self._type.GetValue() == HC.EXPORT_FOLDER_TYPE_SYNCHRONISE:
@ -487,6 +426,8 @@ If you select synchronise, be careful!'''
file_search_context = self._tag_autocomplete.GetFileSearchContext()
metadata_routers = self._metadata_routers_button.GetValue()
run_regularly = self._run_regularly.isChecked()
period = self._period.GetValue()
@ -519,7 +460,7 @@ If you select synchronise, be careful!'''
export_type = export_type,
delete_from_client_after_export = delete_from_client_after_export,
file_search_context = file_search_context,
metadata_routers = self._current_metadata_routers,
metadata_routers = metadata_routers,
run_regularly = run_regularly,
period = period,
phrase = phrase,
@ -550,8 +491,6 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ):
services_manager = HG.client_controller.services_manager
self._neighbouring_txt_tag_service_keys = services_manager.FilterValidServiceKeys( new_options.GetKeyList( 'default_neighbouring_txt_tag_service_keys' ) )
t = ClientGUIListBoxes.ListBoxTagsMedia( self._tags_box, ClientTags.TAG_DISPLAY_ACTUAL, include_counts = True )
self._tags_box.SetTagsBox( t )
@ -585,13 +524,11 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ):
self._export_symlinks = QW.QCheckBox( 'EXPERIMENTAL: export symlinks', self )
self._export_symlinks.setObjectName( 'HydrusWarning' )
text = 'This will export all the files\' tags, newline separated, into .txts beside the files themselves.'
metadata_routers = new_options.GetDefaultExportFilesMetadataRouters()
allowed_importer_classes = [ ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaTags, ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaURLs ]
allowed_exporter_classes = [ ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT, ClientMetadataMigrationExporters.SingleFileMetadataExporterJSON ]
self._export_tag_txts_services_button = ClientGUICommon.BetterButton( self, 'set .txt services', self._SetTxtServices )
self._export_tag_txts = QW.QCheckBox( 'export tags to .txt files?', self )
self._export_tag_txts.setToolTip( text )
self._export_tag_txts.clicked.connect( self.EventExportTagTxtsChanged )
self._metadata_routers_button = ClientGUIMetadataMigration.SingleFileMetadataRoutersButton( self, metadata_routers, allowed_importer_classes, allowed_exporter_classes )
self._export = QW.QPushButton( 'export', self )
self._export.clicked.connect( self._DoExport )
@ -609,11 +546,6 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ):
self._pattern.setText( phrase )
if len( self._neighbouring_txt_tag_service_keys ) > 0:
self._export_tag_txts.setChecked( True )
self._paths.SetData( flat_media )
self._delete_files_after_export.setChecked( HG.client_controller.new_options.GetBoolean( 'delete_files_after_export' ) )
@ -646,11 +578,6 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ):
self._filenames_box.Add( hbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
txt_hbox = QP.HBoxLayout()
QP.AddToLayout( txt_hbox, self._export_tag_txts_services_button, CC.FLAGS_CENTER_PERPENDICULAR )
QP.AddToLayout( txt_hbox, self._export_tag_txts, CC.FLAGS_CENTER_PERPENDICULAR )
vbox = QP.VBoxLayout()
QP.AddToLayout( vbox, top_hbox, CC.FLAGS_EXPAND_SIZER_BOTH_WAYS )
@ -658,18 +585,17 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ):
QP.AddToLayout( vbox, self._filenames_box, CC.FLAGS_EXPAND_PERPENDICULAR )
QP.AddToLayout( vbox, self._delete_files_after_export, CC.FLAGS_ON_RIGHT )
QP.AddToLayout( vbox, self._export_symlinks, CC.FLAGS_ON_RIGHT )
QP.AddToLayout( vbox, txt_hbox, CC.FLAGS_ON_RIGHT )
QP.AddToLayout( vbox, self._metadata_routers_button, CC.FLAGS_ON_RIGHT )
QP.AddToLayout( vbox, self._export, CC.FLAGS_ON_RIGHT )
self.widget().setLayout( vbox )
self._RefreshTags()
self._UpdateTxtButton()
ClientGUIFunctions.SetFocusLater( self._export )
self._paths.itemSelectionChanged.connect( self._RefreshTags )
self._metadata_routers_button.valueChanged.connect( self._MetadataRoutersUpdated )
if do_export_and_then_quit:
@ -781,17 +707,6 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ):
self._RefreshPaths()
export_tag_txts = self._export_tag_txts.isChecked()
if self._export_tag_txts.isChecked():
neighbouring_txt_tag_service_keys = self._neighbouring_txt_tag_service_keys
else:
neighbouring_txt_tag_service_keys = []
directory = self._directory_picker.GetPath()
HydrusPaths.MakeSureDirectoryExists( directory )
@ -811,6 +726,8 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ):
return
metadata_routers = self._metadata_routers_button.GetValue()
client_files_manager = HG.client_controller.client_files_manager
self._export.setEnabled( False )
@ -846,7 +763,7 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ):
def do_it( directory, neighbouring_txt_tag_service_keys, delete_afterwards, export_symlinks, quit_afterwards ):
def do_it( directory, metadata_routers, delete_afterwards, export_symlinks, quit_afterwards ):
job_key = ClientThreading.JobKey( cancellable = True )
@ -856,11 +773,6 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ):
pauser = HydrusData.BigJobPauser()
importers = [ ClientExportingMetadata.SingleFileMetadataImporterExporterMediaTags( service_key ) for service_key in neighbouring_txt_tag_service_keys ]
exporter = ClientExportingMetadata.SingleFileMetadataImporterExporterTXT()
metadata_router = ClientExportingMetadata.SingleFileMetadataRouter( importers = importers, exporter = exporter )
for ( index, ( media, path ) ) in enumerate( to_do ):
number = self._media_to_number_indices[ media ]
@ -893,7 +805,7 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ):
HydrusPaths.MakeSureDirectoryExists( path_dir )
if export_tag_txts:
for metadata_router in metadata_routers:
metadata_router.Work( media.GetMediaResult(), path )
@ -972,7 +884,7 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ):
QP.CallAfter( qt_done, quit_afterwards )
HG.client_controller.CallToThread( do_it, directory, neighbouring_txt_tag_service_keys, delete_afterwards, export_symlinks, quit_afterwards )
HG.client_controller.CallToThread( do_it, directory, metadata_routers, delete_afterwards, export_symlinks, quit_afterwards )
def _GetPath( self, media ):
@ -1002,6 +914,13 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ):
return path
def _MetadataRoutersUpdated( self ):
metadata_routers = self._metadata_routers_button.GetValue()
HG.client_controller.new_options.SetDefaultExportFilesMetadataRouters( metadata_routers )
def _RefreshPaths( self ):
pattern = self._pattern.text()
@ -1035,60 +954,6 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ):
self._tags_box.SetTagsByMedia( flat_media )
def _SetTxtServices( self ):
services_manager = HG.client_controller.services_manager
tag_services = services_manager.GetServices( HC.REAL_TAG_SERVICES )
choice_tuples = [ ( service.GetName(), service.GetServiceKey(), service.GetServiceKey() in self._neighbouring_txt_tag_service_keys ) for service in tag_services ]
try:
neighbouring_txt_tag_service_keys = ClientGUIDialogsQuick.SelectMultipleFromList( self, 'select tag services', choice_tuples )
except HydrusExceptions.CancelledException:
return
self._neighbouring_txt_tag_service_keys = neighbouring_txt_tag_service_keys
HG.client_controller.new_options.SetKeyList( 'default_neighbouring_txt_tag_service_keys', self._neighbouring_txt_tag_service_keys )
if len( self._neighbouring_txt_tag_service_keys ) == 0:
self._export_tag_txts.setChecked( False )
self._UpdateTxtButton()
def _UpdateTxtButton( self ):
if self._export_tag_txts.isChecked():
self._export_tag_txts_services_button.setEnabled( True )
else:
self._export_tag_txts_services_button.setEnabled( False )
if len( self._neighbouring_txt_tag_service_keys ) == 0:
tt = 'No services set.'
else:
names = sorted( [ HG.client_controller.services_manager.GetName( service_key ) for service_key in self._neighbouring_txt_tag_service_keys ] )
tt = ', '.join( names )
self._export_tag_txts_services_button.setToolTip( tt )
def EventExport( self, event ):
self._DoExport()
@ -1106,22 +971,6 @@ class ReviewExportFilesPanel( ClientGUIScrolledPanels.ReviewPanel ):
def EventExportTagTxtsChanged( self ):
turning_on = self._export_tag_txts.isChecked()
self._UpdateTxtButton()
if turning_on:
self._SetTxtServices()
else:
HG.client_controller.new_options.SetKeyList( 'default_neighbouring_txt_tag_service_keys', [] )
def EventOpenLocation( self ):
directory = self._directory_picker.GetPath()

1
hydrus/client/gui/exporting/__init__.py Normal file
View File

@ -0,0 +1 @@

210
hydrus/client/gui/importing/ClientGUIImport.py
View File

@ -27,6 +27,7 @@ from hydrus.client.gui.importing import ClientGUIImportOptions
from hydrus.client.gui.lists import ClientGUIListBoxes
from hydrus.client.gui.lists import ClientGUIListConstants as CGLC
from hydrus.client.gui.lists import ClientGUIListCtrl
from hydrus.client.gui.metadata import ClientGUIMetadataMigration
from hydrus.client.gui.networking import ClientGUINetworkJobControl
from hydrus.client.gui.search import ClientGUIACDropdown
from hydrus.client.gui.widgets import ClientGUICommon
@ -36,6 +37,9 @@ from hydrus.client.importing.options import FileImportOptions
from hydrus.client.importing.options import NoteImportOptions
from hydrus.client.importing.options import TagImportOptions
from hydrus.client.metadata import ClientTags
from hydrus.client.metadata import ClientMetadataMigration
from hydrus.client.metadata import ClientMetadataMigrationExporters
from hydrus.client.metadata import ClientMetadataMigrationImporters
class CheckerOptionsButton( ClientGUICommon.BetterButton ):
@ -477,11 +481,6 @@ class FilenameTaggingOptionsPanel( QW.QWidget ):
self._checkboxes_panel = ClientGUICommon.StaticBox( self, 'misc' )
self._load_from_txt_files_checkbox = QW.QCheckBox( 'try to load tags from neighbouring .txt files', self._checkboxes_panel )
txt_files_help_button = ClientGUICommon.BetterBitmapButton( self._checkboxes_panel, CC.global_pixmaps().help, self._ShowTXTHelp )
txt_files_help_button.setToolTip( 'Show help regarding importing tags from .txt files.' )
self._filename_namespace = QW.QLineEdit( self._checkboxes_panel )
self._filename_namespace.setMinimumWidth( 100 )
@ -510,10 +509,9 @@ class FilenameTaggingOptionsPanel( QW.QWidget ):
#
( tags_for_all, load_from_neighbouring_txt_files, add_filename, directory_dict ) = filename_tagging_options.SimpleToTuple()
( tags_for_all, add_filename, directory_dict ) = filename_tagging_options.SimpleToTuple()
self._tags.AddTags( tags_for_all )
self._load_from_txt_files_checkbox.setChecked( load_from_neighbouring_txt_files )
( add_filename_boolean, add_filename_namespace ) = add_filename
@ -541,17 +539,11 @@ class FilenameTaggingOptionsPanel( QW.QWidget ):
self._single_tags_panel.Add( self._tag_autocomplete_selection, CC.FLAGS_EXPAND_PERPENDICULAR )
self._single_tags_panel.Add( self._single_tags_paste_button, CC.FLAGS_EXPAND_PERPENDICULAR )
txt_hbox = QP.HBoxLayout()
QP.AddToLayout( txt_hbox, self._load_from_txt_files_checkbox, CC.FLAGS_EXPAND_BOTH_WAYS )
QP.AddToLayout( txt_hbox, txt_files_help_button, CC.FLAGS_CENTER_PERPENDICULAR )
filename_hbox = QP.HBoxLayout()
QP.AddToLayout( filename_hbox, self._filename_checkbox, CC.FLAGS_CENTER_PERPENDICULAR )
QP.AddToLayout( filename_hbox, self._filename_namespace, CC.FLAGS_EXPAND_BOTH_WAYS )
self._checkboxes_panel.Add( txt_hbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
self._checkboxes_panel.Add( filename_hbox, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
for index in ( 0, 1, 2, -3, -2, -1 ):
@ -581,7 +573,6 @@ class FilenameTaggingOptionsPanel( QW.QWidget ):
self._tags.tagsRemoved.connect( self.tagsChanged )
self._single_tags.tagsRemoved.connect( self.SingleTagsRemoved )
self._load_from_txt_files_checkbox.clicked.connect( self.tagsChanged )
self._filename_namespace.textChanged.connect( self.tagsChanged )
self._filename_checkbox.clicked.connect( self.tagsChanged )
@ -645,21 +636,6 @@ class FilenameTaggingOptionsPanel( QW.QWidget ):
self.EnterTagsSingle( tags )
def _ShowTXTHelp( self ):
message = 'If you would like to add custom tags with your files, add a .txt file beside the file like so:'
message += os.linesep * 2
message += 'my_file.jpg'
message += os.linesep
message += 'my_file.jpg.txt'
message += os.linesep * 2
message += 'And include your tags inside the .txt file in a newline-separated list (if you know how to script, generating these files automatically from another source of tags can save a lot of time!).'
message += os.linesep * 2
message += 'Make sure you preview the results in the table above to be certain everything is parsing correctly. Until you are comfortable with this, you should test it on just one or two files.'
QW.QMessageBox.information( self, 'Information', message )
def EnterTags( self, tags ):
HG.client_controller.Write( 'push_recent_tags', self._service_key, tags )
@ -756,7 +732,6 @@ class FilenameTaggingOptionsPanel( QW.QWidget ):
def UpdateFilenameTaggingOptions( self, filename_tagging_options ):
tags_for_all = self._tags.GetTags()
load_from_neighbouring_txt_files = self._load_from_txt_files_checkbox.isChecked()
add_filename = ( self._filename_checkbox.isChecked(), self._filename_namespace.text() )
@ -767,7 +742,7 @@ class FilenameTaggingOptionsPanel( QW.QWidget ):
directories_dict[ index ] = ( dir_checkbox.isChecked(), dir_namespace_textctrl.text() )
filename_tagging_options.SimpleSetTuple( tags_for_all, load_from_neighbouring_txt_files, add_filename, directories_dict )
filename_tagging_options.SimpleSetTuple( tags_for_all, add_filename, directories_dict )
@ -776,17 +751,27 @@ class EditLocalImportFilenameTaggingPanel( ClientGUIScrolledPanels.EditPanel ):
def __init__( self, parent, paths ):
# TODO: a really nice rewrite for all this, perhaps when I go for string conversions here, would be to eliminate the multi-page format and instead update the controls.
# changing service while maintaining focus and list selection would be great
# an option here is to mutate the sidecars a little and just have a 'filename' source. this could wangle everything neatly and send to whatever service
# but only if the UI can stay helpful. maybe we shouldn't replace the easy UI, but we can replace the guts behind the scenes with metadata routers
# however, changing service while maintaining focus and list selection would be great
ClientGUIScrolledPanels.EditPanel.__init__( self, parent )
self._paths = paths
self._filename_tagging_option_pages = []
self._tag_repositories = ClientGUICommon.BetterNotebook( self )
self._notebook = ClientGUICommon.BetterNotebook( self )
#
# TODO: could have default import here and favourites system
metadata_routers = []
self._metadata_router_page = self._MetadataRoutersPanel( self._notebook, metadata_routers, paths )
self._notebook.addTab( self._metadata_router_page, 'sidecars' )
services = HG.client_controller.services_manager.GetServices( HC.REAL_TAG_SERVICES )
default_tag_service_key = HG.client_controller.new_options.GetKey( 'default_tag_service_tab' )
@ -796,18 +781,20 @@ class EditLocalImportFilenameTaggingPanel( ClientGUIScrolledPanels.EditPanel ):
service_key = service.GetServiceKey()
name = service.GetName()
page = self._Panel( self._tag_repositories, service_key, paths )
page = self._FilenameTaggingOptionsPanel( self._notebook, service_key, paths )
self._filename_tagging_option_pages.append( page )
page.movePageLeft.connect( self.MovePageLeft )
page.movePageRight.connect( self.MovePageRight )
select = service_key == default_tag_service_key
tab_index = self._tag_repositories.addTab( page, name )
tab_index = self._notebook.addTab( page, name )
if select:
self._tag_repositories.setCurrentIndex( tab_index )
self._notebook.setCurrentIndex( tab_index )
@ -815,39 +802,44 @@ class EditLocalImportFilenameTaggingPanel( ClientGUIScrolledPanels.EditPanel ):
vbox = QP.VBoxLayout()
QP.AddToLayout( vbox, self._tag_repositories, CC.FLAGS_EXPAND_BOTH_WAYS )
QP.AddToLayout( vbox, self._notebook, CC.FLAGS_EXPAND_BOTH_WAYS )
self.widget().setLayout( vbox )
self._tag_repositories.currentChanged.connect( self._SaveDefaultTagServiceKey )
self._notebook.currentChanged.connect( self._SaveDefaultTagServiceKey )
self._tag_repositories.currentWidget().SetSearchFocus()
self._notebook.currentWidget().SetSearchFocus()
def _SaveDefaultTagServiceKey( self ):
if self.sender() != self._tag_repositories:
if self.sender() != self._notebook:
return
if HG.client_controller.new_options.GetBoolean( 'save_default_tag_service_tab_on_change' ):
current_page = self._tag_repositories.currentWidget()
current_page = self._notebook.currentWidget()
HG.client_controller.new_options.SetKey( 'default_tag_service_tab', current_page.GetServiceKey() )
if current_page in self._filename_tagging_option_pages:
HG.client_controller.new_options.SetKey( 'default_tag_service_tab', current_page.GetServiceKey() )
def GetValue( self ):
metadata_routers = self._metadata_router_page.GetValue()
paths_to_additional_service_keys_to_tags = collections.defaultdict( ClientTags.ServiceKeysToTags )
for page in self._tag_repositories.GetPages():
for page in self._filename_tagging_option_pages:
( service_key, page_of_paths_to_tags ) = page.GetInfo()
( service_key, paths_to_tags ) = page.GetInfo()
for ( path, tags ) in page_of_paths_to_tags.items():
for ( path, tags ) in paths_to_tags.items():
if len( tags ) == 0:
@ -858,24 +850,24 @@ class EditLocalImportFilenameTaggingPanel( ClientGUIScrolledPanels.EditPanel ):
return paths_to_additional_service_keys_to_tags
return ( metadata_routers, paths_to_additional_service_keys_to_tags )
def MovePageLeft( self ):
self._tag_repositories.SelectLeft()
self._notebook.SelectLeft()
self._tag_repositories.currentWidget().SetSearchFocus()
self._notebook.currentWidget().SetSearchFocus()
def MovePageRight( self ):
self._tag_repositories.SelectRight()
self._notebook.SelectRight()
self._tag_repositories.currentWidget().SetSearchFocus()
self._notebook.currentWidget().SetSearchFocus()
class _Panel( QW.QWidget ):
class _FilenameTaggingOptionsPanel( QW.QWidget ):
movePageLeft = QC.Signal()
movePageRight = QC.Signal()
@ -989,6 +981,122 @@ class EditLocalImportFilenameTaggingPanel( ClientGUIScrolledPanels.EditPanel ):
class _MetadataRoutersPanel( QW.QWidget ):
def __init__( self, parent, metadata_routers, paths ):
QW.QWidget.__init__( self, parent )
self._paths = paths
self._paths_list = ClientGUIListCtrl.BetterListCtrl( self, CGLC.COLUMN_LIST_PATHS_TO_TAGS.ID, 10, self._ConvertDataToListCtrlTuples )
allowed_importer_classes = [ ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT, ClientMetadataMigrationImporters.SingleFileMetadataImporterJSON ]
allowed_exporter_classes = [ ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags, ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaURLs ]
self._metadata_routers_panel = ClientGUIMetadataMigration.SingleFileMetadataRoutersControl( self, metadata_routers, allowed_importer_classes, allowed_exporter_classes )
#
self._schedule_refresh_file_list_job = None
#
# i.e. ( index, path )
self._paths_list.AddDatas( list( enumerate( self._paths ) ) )
#
vbox = QP.VBoxLayout()
QP.AddToLayout( vbox, self._paths_list, CC.FLAGS_EXPAND_BOTH_WAYS )
QP.AddToLayout( vbox, self._metadata_routers_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
self.setLayout( vbox )
self._metadata_routers_panel.listBoxChanged.connect( self.ScheduleRefreshFileList )
def _ConvertDataToListCtrlTuples( self, data ):
( index, path ) = data
strings = self._GetStrings( path )
pretty_index = HydrusData.ToHumanInt( index + 1 )
pretty_path = path
pretty_strings = ', '.join( strings )
display_tuple = ( pretty_index, pretty_path, pretty_strings )
sort_tuple = ( index, path, strings )
return ( display_tuple, sort_tuple )
def _GetStrings( self, path ):
strings = []
metadata_routers = self._metadata_routers_panel.GetValue()
for router in metadata_routers:
pre_processed_strings = set()
for importer in router.GetImporters():
if isinstance( importer, ClientMetadataMigrationImporters.SingleFileMetadataImporterSidecar ):
pre_processed_strings.update( importer.Import( path ) )
else:
continue
if len( pre_processed_strings ) == 0:
continue
processed_strings = router.GetStringProcessor().ProcessStrings( pre_processed_strings )
strings.extend( sorted( processed_strings ) )
return strings
def GetValue( self ):
return self._metadata_routers_panel.GetValue()
def RefreshFileList( self ):
self._paths_list.UpdateDatas()
def ScheduleRefreshFileList( self ):
if self._schedule_refresh_file_list_job is not None:
self._schedule_refresh_file_list_job.Cancel()
self._schedule_refresh_file_list_job = None
self._schedule_refresh_file_list_job = HG.client_controller.CallLaterQtSafe( self, 0.5, 'refresh path list', self.RefreshFileList )
def SetSearchFocus( self ):
pass
class EditFilenameTaggingOptionPanel( ClientGUIScrolledPanels.EditPanel ):
def __init__( self, parent, service_key, filename_tagging_options ):

21
hydrus/client/gui/importing/ClientGUIImportFolders.py
View File

@ -18,9 +18,13 @@ from hydrus.client.gui.importing import ClientGUIImport
from hydrus.client.gui.importing import ClientGUIImportOptions
from hydrus.client.gui.lists import ClientGUIListConstants as CGLC
from hydrus.client.gui.lists import ClientGUIListCtrl
from hydrus.client.gui.metadata import ClientGUIMetadataMigration
from hydrus.client.gui.widgets import ClientGUICommon
from hydrus.client.importing import ClientImportLocal
from hydrus.client.importing.options import TagImportOptions
from hydrus.client.metadata import ClientMetadataMigration
from hydrus.client.metadata import ClientMetadataMigrationExporters
from hydrus.client.metadata import ClientMetadataMigrationImporters
class EditImportFoldersPanel( ClientGUIScrolledPanels.EditPanel ):
@ -170,7 +174,7 @@ class EditImportFoldersPanel( ClientGUIScrolledPanels.EditPanel ):
class EditImportFolderPanel( ClientGUIScrolledPanels.EditPanel ):
def __init__( self, parent, import_folder ):
def __init__( self, parent, import_folder: ClientImportLocal.ImportFolder ):
ClientGUIScrolledPanels.EditPanel.__init__( self, parent )
@ -240,7 +244,7 @@ class EditImportFolderPanel( ClientGUIScrolledPanels.EditPanel ):
#
self._filename_tagging_options_box = ClientGUICommon.StaticBox( self, 'filename tagging' )
self._filename_tagging_options_box = ClientGUICommon.StaticBox( self, 'metadata import' )
filename_tagging_options_panel = ClientGUIListCtrl.BetterListCtrlPanel( self._filename_tagging_options_box )
@ -252,6 +256,12 @@ class EditImportFolderPanel( ClientGUIScrolledPanels.EditPanel ):
filename_tagging_options_panel.AddButton( 'edit', self._EditFilenameTaggingOptions, enabled_only_on_selection = True )
filename_tagging_options_panel.AddDeleteButton()
metadata_routers = self._import_folder.GetMetadataRouters()
allowed_importer_classes = [ ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT, ClientMetadataMigrationImporters.SingleFileMetadataImporterJSON ]
allowed_exporter_classes = [ ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags, ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaURLs ]
self._metadata_routers_button = ClientGUIMetadataMigration.SingleFileMetadataRoutersButton( self, metadata_routers, allowed_importer_classes, allowed_exporter_classes )
services_manager = HG.client_controller.services_manager
#
@ -343,7 +353,10 @@ class EditImportFolderPanel( ClientGUIScrolledPanels.EditPanel ):
#
self._filename_tagging_options_box.Add( ClientGUICommon.BetterStaticText( self._filename_tagging_options_box, 'filename tagging:' ), CC.FLAGS_CENTER_PERPENDICULAR )
self._filename_tagging_options_box.Add( filename_tagging_options_panel, CC.FLAGS_EXPAND_BOTH_WAYS )
self._filename_tagging_options_box.Add( ClientGUICommon.BetterStaticText( self._filename_tagging_options_box, 'sidecar importing:' ), CC.FLAGS_CENTER_PERPENDICULAR )
self._filename_tagging_options_box.Add( self._metadata_routers_button, CC.FLAGS_EXPAND_PERPENDICULAR )
#
@ -632,5 +645,9 @@ class EditImportFolderPanel( ClientGUIScrolledPanels.EditPanel ):
self._import_folder.SetTuple( name, path, file_import_options, tag_import_options, tag_service_keys_to_filename_tagging_options, actions, action_locations, period, check_regularly, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page )
metadata_routers = self._metadata_routers_button.GetValue()
self._import_folder.SetMetadataRouters( metadata_routers )
return self._import_folder

5
hydrus/client/gui/lists/ClientGUIListBoxes.py
View File

@ -920,6 +920,11 @@ class QueueListBox( QW.QWidget ):
self.listBoxChanged.emit()
def Clear( self ):
self._listbox.clear()
def GetCount( self ):
return self._listbox.count()

242
hydrus/client/gui/metadata/ClientGUIMetadataMigration.py Normal file
View File

@ -0,0 +1,242 @@
import typing
from qtpy import QtCore as QC
from qtpy import QtWidgets as QW
from hydrus.core import HydrusData
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusText
from hydrus.client import ClientConstants as CC
from hydrus.client import ClientParsing
from hydrus.client.gui import ClientGUIFunctions
from hydrus.client.gui import ClientGUIScrolledPanels
from hydrus.client.gui import ClientGUIStringControls
from hydrus.client.gui import ClientGUITopLevelWindowsPanels
from hydrus.client.gui import QtPorting as QP
from hydrus.client.gui.lists import ClientGUIListBoxes
from hydrus.client.gui.metadata import ClientGUIMetadataMigrationExporters
from hydrus.client.gui.metadata import ClientGUIMetadataMigrationImporters
from hydrus.client.gui.widgets import ClientGUICommon
from hydrus.client.metadata import ClientMetadataMigration
class EditSingleFileMetadataRouterPanel( ClientGUIScrolledPanels.EditPanel ):
def __init__( self, parent: QW.QWidget, router: ClientMetadataMigration.SingleFileMetadataRouter, allowed_importer_classes: list, allowed_exporter_classes: list ):
ClientGUIScrolledPanels.EditPanel.__init__( self, parent )
self._original_router = router
self._allowed_importer_classes = allowed_importer_classes
self._allowed_exporter_classes = allowed_exporter_classes
importers = self._original_router.GetImporters()
string_processor = self._original_router.GetStringProcessor()
exporter = self._original_router.GetExporter()
#
self._importers_panel = ClientGUICommon.StaticBox( self, 'sources' )
self._importers_list = ClientGUIMetadataMigrationImporters.SingleFileMetadataImportersControl( self._importers_panel, importers, self._allowed_importer_classes )
self._importers_panel.Add( self._importers_list, CC.FLAGS_EXPAND_BOTH_WAYS )
#
self._processing_panel = ClientGUICommon.StaticBox( self, 'processing' )
self._string_processor_button = ClientGUIStringControls.StringProcessorButton( self._processing_panel, string_processor, self._GetExampleStringProcessorTestData )
st = ClientGUICommon.BetterStaticText( self._processing_panel, 'You can alter all the texts before export here.' )
self._processing_panel.Add( st, CC.FLAGS_EXPAND_PERPENDICULAR )
self._processing_panel.Add( self._string_processor_button, CC.FLAGS_EXPAND_PERPENDICULAR )
#
self._exporter_panel = ClientGUICommon.StaticBox( self, 'destination' )
self._exporter_button = ClientGUIMetadataMigrationExporters.EditSingleFileMetadataExporterPanel( self._exporter_panel, exporter, self._allowed_exporter_classes )
self._exporter_panel.Add( self._exporter_button, CC.FLAGS_EXPAND_BOTH_WAYS )
#
vbox = QP.VBoxLayout()
QP.AddToLayout( vbox, self._importers_panel, CC.FLAGS_EXPAND_BOTH_WAYS )
QP.AddToLayout( vbox, self._processing_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
QP.AddToLayout( vbox, self._exporter_panel, CC.FLAGS_EXPAND_BOTH_WAYS )
self.widget().setLayout( vbox )
def _GetExampleStringProcessorTestData( self ):
example_parsing_context = dict()
importers = self._importers_list.GetData()
exporter = self._exporter_button.GetValue()
texts = set()
for importer in importers:
texts.update( importer.GetExampleStrings() )
texts.update( exporter.GetExampleStrings() )
texts = sorted( texts )
return ClientParsing.ParsingTestData( example_parsing_context, texts )
def _GetValue( self ) -> ClientMetadataMigration.SingleFileMetadataRouter:
importers = self._importers_list.GetData()
string_processor = self._string_processor_button.GetValue()
exporter = self._exporter_button.GetValue()
router = ClientMetadataMigration.SingleFileMetadataRouter( importers = importers, string_processor = string_processor, exporter = exporter )
return router
def GetValue( self ) -> ClientMetadataMigration.SingleFileMetadataRouter:
router = self._GetValue()
return router
def convert_router_to_pretty_string( router: ClientMetadataMigration.SingleFileMetadataRouter ) -> str:
return router.ToString( pretty = True )
class SingleFileMetadataRoutersControl( ClientGUIListBoxes.AddEditDeleteListBox ):
def __init__( self, parent: QW.QWidget, routers: typing.Collection[ ClientMetadataMigration.SingleFileMetadataRouter ], allowed_importer_classes: list, allowed_exporter_classes: list ):
ClientGUIListBoxes.AddEditDeleteListBox.__init__( self, parent, 5, convert_router_to_pretty_string, self._AddRouter, self._EditRouter )
self._allowed_importer_classes = allowed_importer_classes
self._allowed_exporter_classes = allowed_exporter_classes
self.AddDatas( routers )
width = ClientGUIFunctions.ConvertTextToPixelWidth( self, 64 )
self.setMinimumWidth( width )
def _AddRouter( self ):
exporter = self._allowed_exporter_classes[0]()
router = ClientMetadataMigration.SingleFileMetadataRouter( exporter = exporter )
return self._EditRouter( router )
def _EditRouter( self, router: ClientMetadataMigration.SingleFileMetadataRouter ):
with ClientGUITopLevelWindowsPanels.DialogEdit( self, 'edit metadata migration router' ) as dlg:
panel = EditSingleFileMetadataRouterPanel( self, router, self._allowed_importer_classes, self._allowed_exporter_classes )
dlg.SetPanel( panel )
if dlg.exec() == QW.QDialog.Accepted:
edited_router = panel.GetValue()
return edited_router
raise HydrusExceptions.VetoException()
class SingleFileMetadataRoutersButton( QW.QPushButton ):
valueChanged = QC.Signal()
def __init__( self, parent: QW.QWidget, routers: typing.Collection[ ClientMetadataMigration.SingleFileMetadataRouter ], allowed_importer_classes: list, allowed_exporter_classes: list ):
QW.QPushButton.__init__( self, parent )
self._routers = routers
self._allowed_importer_classes = allowed_importer_classes
self._allowed_exporter_classes = allowed_exporter_classes
self._RefreshLabel()
self.clicked.connect( self._Edit )
def _Edit( self ):
with ClientGUITopLevelWindowsPanels.DialogEdit( self, 'edit metadata migration routers' ) as dlg:
panel = ClientGUIScrolledPanels.EditSingleCtrlPanel( dlg )
control = SingleFileMetadataRoutersControl( panel, self._routers, self._allowed_importer_classes, self._allowed_exporter_classes )
panel.SetControl( control )
dlg.SetPanel( panel )
if dlg.exec() == QW.QDialog.Accepted:
value = control.GetData()
self.SetValue( value )
self.valueChanged.emit()
def _RefreshLabel( self ):
if len( self._routers ) == 0:
text = 'no metadata migration'
elif len( self._routers ) == 1:
( router, ) = self._routers
text = router.ToString( pretty = True )
else:
text = '{} metadata migrations'.format( HydrusData.ToHumanInt( len( self._routers ) ) )
elided_text = HydrusText.ElideText( text, 64 )
self.setText( elided_text )
self.setToolTip( text )
def GetValue( self ):
return self._routers
def SetValue( self, routers: typing.Collection[ ClientMetadataMigration.SingleFileMetadataRouter ] ):
self._routers = routers
self._RefreshLabel()

368
hydrus/client/gui/metadata/ClientGUIMetadataMigrationExporters.py Normal file
View File

@ -0,0 +1,368 @@
from qtpy import QtCore as QC
from qtpy import QtWidgets as QW
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusGlobals as HG
from hydrus.core import HydrusText
from hydrus.client import ClientConstants as CC
from hydrus.client import ClientParsing
from hydrus.client.gui import ClientGUIDialogs
from hydrus.client.gui import ClientGUIDialogsQuick
from hydrus.client.gui import ClientGUIScrolledPanels
from hydrus.client.gui import ClientGUITopLevelWindowsPanels
from hydrus.client.gui import QtPorting as QP
from hydrus.client.gui.lists import ClientGUIListBoxes
from hydrus.client.gui.widgets import ClientGUICommon
from hydrus.client.metadata import ClientMetadataMigrationExporters
choice_tuple_label_lookup = {
ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags : 'a file\'s tags',
ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaURLs : 'a file\'s URLs',
ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT : 'a .txt sidecar',
ClientMetadataMigrationExporters.SingleFileMetadataExporterJSON : 'a .json sidecar'
}
choice_tuple_description_lookup = {
ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags : 'The tags that a file has on a particular service.',
ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaURLs : 'The known URLs that a file has.',
ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT : 'A list of raw newline-separated texts in a .txt file.',
ClientMetadataMigrationExporters.SingleFileMetadataExporterJSON : 'Strings somewhere in a JSON file.'
}
def SelectClass( win: QW.QWidget, allowed_exporter_classes: list ):
choice_tuples = [ ( choice_tuple_label_lookup[ c ], c, choice_tuple_description_lookup[ c ] ) for c in allowed_exporter_classes ]
message = 'Which kind of destination are we going to use?'
exporter_class = ClientGUIDialogsQuick.SelectFromListButtons( win, 'Which type?', choice_tuples, message = message )
return exporter_class
class EditSingleFileMetadataExporterPanel( ClientGUIScrolledPanels.EditPanel ):
def __init__( self, parent: QW.QWidget, exporter: ClientMetadataMigrationExporters.SingleFileMetadataExporter, allowed_exporter_classes: list ):
ClientGUIScrolledPanels.EditPanel.__init__( self, parent )
self._original_exporter = exporter
self._allowed_exporter_classes = allowed_exporter_classes
self._current_exporter_class = type( exporter )
self._service_key = CC.COMBINED_TAG_SERVICE_KEY
#
self._change_type_button = ClientGUICommon.BetterButton( self, 'change type', self._ChangeType )
#
self._service_selection_panel = QW.QWidget( self )
self._service_selection_button = ClientGUICommon.BetterButton( self._service_selection_panel, 'service', self._SelectService )
hbox = ClientGUICommon.WrapInText( self._service_selection_button, self._service_selection_panel, 'tag service: ' )
self._service_selection_panel.setLayout( hbox )
#
self._nested_object_names_panel = QW.QWidget( self )
self._nested_object_names_list = ClientGUIListBoxes.QueueListBox( self, 6, str, self._AddObjectName, self._EditObjectName )
tt = 'If you set this as [files,tags], the exported strings will be placed under the nested objects with keys "files"->"tags". Note that this will also update an existing file, so, if you are feeling clever, you can have multiple routers writing tags and URLs to different locations in the same file!'
self._nested_object_names_list.setToolTip( tt )
vbox = QP.VBoxLayout()
QP.AddToLayout( vbox, ClientGUICommon.BetterStaticText( self._nested_object_names_panel, 'JSON Objects structure' ), CC.FLAGS_EXPAND_PERPENDICULAR )
QP.AddToLayout( vbox, self._nested_object_names_list, CC.FLAGS_EXPAND_BOTH_WAYS )
self._nested_object_names_panel.setLayout( vbox )
#
self._suffix_panel = QW.QWidget( self )
self._suffix = QW.QLineEdit( self )
tt = 'If you set this to "tags", the exported filename will be (file filename).tags.ext, where ext is .txt/.json/.xml etc... . Leave blank to just export to (file filename).ext.'
self._suffix.setToolTip( tt )
hbox = ClientGUICommon.WrapInText( self._suffix, self._suffix_panel, 'filename suffix: ' )
self._suffix_panel.setLayout( hbox )
#
vbox = QP.VBoxLayout()
QP.AddToLayout( vbox, self._change_type_button, CC.FLAGS_EXPAND_PERPENDICULAR )
QP.AddToLayout( vbox, self._service_selection_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
QP.AddToLayout( vbox, self._nested_object_names_panel, CC.FLAGS_EXPAND_BOTH_WAYS )
QP.AddToLayout( vbox, self._suffix_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
vbox.addStretch( 1 )
self.widget().setLayout( vbox )
self._SetValue( exporter )
def _AddObjectName( self ):
object_name = ''
return self._EditObjectName( object_name )
def _ChangeType( self ):
allowed_exporter_classes = list( self._allowed_exporter_classes )
if self._current_exporter_class in allowed_exporter_classes:
allowed_exporter_classes.remove( self._current_exporter_class )
if len( allowed_exporter_classes ) == 0:
message = 'Sorry, you can only have this one!'
QW.QMessageBox.information( self, 'Information', message )
try:
exporter_class = SelectClass( self, allowed_exporter_classes )
except HydrusExceptions.CancelledException:
return
exporter = exporter_class()
# it is nice to preserve old values as we flip from one type to another. more pleasant that making the user cancel and re-open
if isinstance( exporter, ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags ):
exporter.SetServiceKey( self._service_key )
elif isinstance( exporter, ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaURLs ):
pass
elif isinstance( exporter, ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT ):
exporter.SetSuffix( self._suffix.text() )
elif isinstance( exporter, ClientMetadataMigrationExporters.SingleFileMetadataExporterJSON ):
exporter.SetSuffix( self._suffix.text() )
exporter.SetNestedObjectNames( self._nested_object_names_list.GetData() )
self._SetValue( exporter )
def _EditObjectName( self, object_name ):
with ClientGUIDialogs.DialogTextEntry( self, 'enter the JSON Object name', default = object_name, allow_blank = False ) as dlg:
if dlg.exec() == QW.QDialog.Accepted:
object_name = dlg.GetValue()
return object_name
else:
raise HydrusExceptions.VetoException()
def _GetExampleTestData( self ):
example_parsing_context = dict()
exporter = self._GetValue()
texts = sorted( exporter.GetExampleStrings() )
return ClientParsing.ParsingTestData( example_parsing_context, texts )
def _GetValue( self ) -> ClientMetadataMigrationExporters.SingleFileMetadataExporter:
if self._current_exporter_class == ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags:
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags( service_key = self._service_key )
elif self._current_exporter_class == ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaURLs:
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaURLs()
elif self._current_exporter_class == ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT:
suffix = self._suffix.text()
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT( suffix = suffix )
elif self._current_exporter_class == ClientMetadataMigrationExporters.SingleFileMetadataExporterJSON:
suffix = self._suffix.text()
nested_object_names = self._nested_object_names_list.GetData()
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterJSON( suffix = suffix, nested_object_names = nested_object_names )
else:
raise Exception( 'Did not understand the current exporter type!' )
return exporter
def _SelectService( self ):
service_key = ClientGUIDialogsQuick.SelectServiceKey( service_types = HC.ALL_TAG_SERVICES, unallowed = [ self._service_key ] )
if service_key is None:
return
self._service_key = service_key
self._UpdateServiceKeyButtonLabel()
def _SetValue( self, exporter: ClientMetadataMigrationExporters.SingleFileMetadataExporter ):
self._current_exporter_class = type( exporter )
self._change_type_button.setText( choice_tuple_label_lookup[ self._current_exporter_class ] )
self._service_selection_panel.setVisible( False )
self._nested_object_names_panel.setVisible( False )
self._suffix_panel.setVisible( False )
if isinstance( exporter, ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags ):
self._service_key = exporter.GetServiceKey()
self._UpdateServiceKeyButtonLabel()
self._service_selection_panel.setVisible( True )
elif isinstance( exporter, ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaURLs ):
pass
elif isinstance( exporter, ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT ):
suffix = exporter.GetSuffix()
self._suffix.setText( suffix )
self._suffix_panel.setVisible( True )
elif isinstance( exporter, ClientMetadataMigrationExporters.SingleFileMetadataExporterJSON ):
suffix = exporter.GetSuffix()
self._suffix.setText( suffix )
self._suffix_panel.setVisible( True )
nested_object_names = exporter.GetNestedObjectNames()
self._nested_object_names_list.Clear()
self._nested_object_names_list.AddDatas( nested_object_names )
self._nested_object_names_panel.setVisible( True )
else:
raise Exception( 'Did not understand the new exporter type!' )
def _UpdateServiceKeyButtonLabel( self ):
name = HG.client_controller.services_manager.GetName( self._service_key )
self._service_selection_button.setText( name )
def GetValue( self ) -> ClientMetadataMigrationExporters.SingleFileMetadataExporter:
exporter = self._GetValue()
return exporter
class SingleFileMetadataExporterButton( QW.QPushButton ):
valueChanged = QC.Signal()
def __init__( self, parent: QW.QWidget, exporter: ClientMetadataMigrationExporters.SingleFileMetadataExporter, allowed_exporter_classes: list ):
QW.QPushButton.__init__( self, parent )
self._exporter = exporter
self._allowed_exporter_classes = allowed_exporter_classes
self._RefreshLabel()
self.clicked.connect( self._Edit )
def _Edit( self ):
with ClientGUITopLevelWindowsPanels.DialogEdit( self, 'edit metadata migration exporter' ) as dlg:
panel = EditSingleFileMetadataExporterPanel( dlg, self._exporter, self._allowed_exporter_classes )
dlg.SetPanel( panel )
if dlg.exec() == QW.QDialog.Accepted:
value = panel.GetValue()
self.SetValue( value )
self.valueChanged.emit()
def _RefreshLabel( self ):
text = self._exporter.ToString()
elided_text = HydrusText.ElideText( text, 64 )
self.setText( elided_text )
self.setToolTip( text )
def GetValue( self ):
return self._exporter
def SetValue( self, exporter: ClientMetadataMigrationExporters.SingleFileMetadataExporter ):
self._exporter = exporter
self._RefreshLabel()

394
hydrus/client/gui/metadata/ClientGUIMetadataMigrationImporters.py Normal file
View File

@ -0,0 +1,394 @@
import json
import typing
from qtpy import QtWidgets as QW
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusGlobals as HG
from hydrus.client import ClientConstants as CC
from hydrus.client import ClientParsing
from hydrus.client import ClientStrings
from hydrus.client.gui import ClientGUIDialogsQuick
from hydrus.client.gui import ClientGUIScrolledPanels
from hydrus.client.gui import ClientGUIStringControls
from hydrus.client.gui import ClientGUITopLevelWindowsPanels
from hydrus.client.gui import QtPorting as QP
from hydrus.client.gui.lists import ClientGUIListBoxes
from hydrus.client.gui.parsing import ClientGUIParsingFormulae
from hydrus.client.gui.widgets import ClientGUICommon
from hydrus.client.metadata import ClientMetadataMigrationImporters
choice_tuple_label_lookup = {
ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaTags : 'a file\'s tags',
ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaURLs : 'a file\'s URLs',
ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT : 'a .txt sidecar',
ClientMetadataMigrationImporters.SingleFileMetadataImporterJSON : 'a .json sidecar'
}
choice_tuple_description_lookup = {
ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaTags : 'The tags that a file has on a particular service.',
ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaURLs : 'The known URLs that a file has.',
ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT : 'A list of raw newline-separated texts in a .txt file.',
ClientMetadataMigrationImporters.SingleFileMetadataImporterJSON : 'Strings somewhere in a JSON file.'
}
def SelectClass( win: QW.QWidget, allowed_importer_classes: list ):
choice_tuples = [ ( choice_tuple_label_lookup[ c ], c, choice_tuple_description_lookup[ c ] ) for c in allowed_importer_classes ]
message = 'Which kind of source are we going to use?'
importer_class = ClientGUIDialogsQuick.SelectFromListButtons( win, 'Which type?', choice_tuples, message = message )
return importer_class
class EditSingleFileMetadataImporterPanel( ClientGUIScrolledPanels.EditPanel ):
def __init__( self, parent: QW.QWidget, importer: ClientMetadataMigrationImporters.SingleFileMetadataImporter, allowed_importer_classes: list ):
ClientGUIScrolledPanels.EditPanel.__init__( self, parent )
self._original_importer = importer
self._allowed_importer_classes = allowed_importer_classes
self._current_importer_class = type( importer )
self._service_key = CC.COMBINED_TAG_SERVICE_KEY
self._json_parsing_formula = ClientParsing.ParseFormulaJSON()
string_processor = importer.GetStringProcessor()
#
self._change_type_button = ClientGUICommon.BetterButton( self, 'change type', self._ChangeType )
#
self._service_selection_panel = QW.QWidget( self )
self._service_selection_button = ClientGUICommon.BetterButton( self, 'service', self._SelectService )
hbox = ClientGUICommon.WrapInText( self._service_selection_button, self._service_selection_panel, 'tag service: ' )
self._service_selection_panel.setLayout( hbox )
#
self._json_parsing_formula_panel = QW.QWidget( self )
self._json_parsing_formula_button = ClientGUICommon.BetterButton( self, 'edit parsing formula', self._EditJSONParsingFormula )
hbox = ClientGUICommon.WrapInText( self._json_parsing_formula_button, self._json_parsing_formula_panel, 'json parsing formula: ' )
self._json_parsing_formula_panel.setLayout( hbox )
#
self._suffix_panel = QW.QWidget( self )
self._suffix = QW.QLineEdit( self )
tt = 'If you set this to "tags", the filename to import from will be (file filename).tags.ext, where ext is .txt/.json/.xml etc... . Leave blank to just read (file filename).ext.'
self._suffix.setToolTip( tt )
hbox = ClientGUICommon.WrapInText( self._suffix, self._suffix_panel, 'filename suffix: ' )
self._suffix_panel.setLayout( hbox )
#
self._string_processor_panel = QW.QWidget( self )
self._string_processor_button = ClientGUIStringControls.StringProcessorButton( self, string_processor, self._GetExampleTestData )
tt = 'You can alter the texts that come in through this source here.'
self._string_processor_button.setToolTip( tt )
vbox = QP.VBoxLayout()
QP.AddToLayout( vbox, ClientGUICommon.BetterStaticText( self._string_processor_panel, 'You can alter the texts that come in through this source here.' ), CC.FLAGS_EXPAND_PERPENDICULAR )
QP.AddToLayout( vbox, self._string_processor_button, CC.FLAGS_EXPAND_PERPENDICULAR )
self._string_processor_panel.setLayout( vbox )
#
vbox = QP.VBoxLayout()
QP.AddToLayout( vbox, self._change_type_button, CC.FLAGS_EXPAND_PERPENDICULAR )
QP.AddToLayout( vbox, self._service_selection_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
QP.AddToLayout( vbox, self._json_parsing_formula_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
QP.AddToLayout( vbox, self._suffix_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
QP.AddToLayout( vbox, self._string_processor_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
vbox.addStretch( 1 )
self.widget().setLayout( vbox )
self._SetValue( importer )
def _ChangeType( self ):
allowed_importer_classes = list( self._allowed_importer_classes )
if self._current_importer_class in allowed_importer_classes:
allowed_importer_classes.remove( self._current_importer_class )
if len( allowed_importer_classes ) == 0:
message = 'Sorry, you can only have this one!'
QW.QMessageBox.information( self, 'Information', message )
try:
importer_class = SelectClass( self, allowed_importer_classes )
except HydrusExceptions.CancelledException:
return
string_processor = self._string_processor_button.GetValue()
importer = importer_class( string_processor )
# it is nice to preserve old values as we flip from one type to another. more pleasant that making the user cancel and re-open
if isinstance( importer, ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaTags ):
importer.SetServiceKey( self._service_key )
elif isinstance( importer, ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaURLs ):
pass
elif isinstance( importer, ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT ):
importer.SetSuffix( self._suffix.text() )
elif isinstance( importer, ClientMetadataMigrationImporters.SingleFileMetadataImporterJSON ):
importer.SetSuffix( self._suffix.text() )
importer.SetJSONParsingFormula( self._json_parsing_formula )
self._SetValue( importer )
def _EditJSONParsingFormula( self ):
test_data = self._GetExampleTestData()
dlg_title = 'edit formula'
with ClientGUITopLevelWindowsPanels.DialogEdit( self, dlg_title, frame_key = 'deeply_nested_dialog' ) as dlg:
collapse_newlines = False
panel = ClientGUIParsingFormulae.EditJSONFormulaPanel( dlg, collapse_newlines, self._json_parsing_formula, test_data )
dlg.SetPanel( panel )
if dlg.exec() == QW.QDialog.Accepted:
self._json_parsing_formula = panel.GetValue()
def _GetExampleTestData( self ):
example_parsing_context = dict()
importer = self._GetValue()
texts = sorted( importer.GetExampleStrings() )
return ClientParsing.ParsingTestData( example_parsing_context, [ json.dumps( texts ) ] )
def _GetValue( self ) -> ClientMetadataMigrationImporters.SingleFileMetadataImporter:
string_processor = self._string_processor_button.GetValue()
if self._current_importer_class == ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaTags:
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaTags( string_processor = string_processor, service_key = self._service_key )
elif self._current_importer_class == ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaURLs:
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaURLs( string_processor = string_processor )
elif self._current_importer_class == ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT:
suffix = self._suffix.text()
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT( string_processor = string_processor, suffix = suffix )
elif self._current_importer_class == ClientMetadataMigrationImporters.SingleFileMetadataImporterJSON:
suffix = self._suffix.text()
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterJSON( string_processor = string_processor, suffix = suffix, json_parsing_formula = self._json_parsing_formula )
else:
raise Exception( 'Did not understand the current importer type!' )
return importer
def _SelectService( self ):
service_key = ClientGUIDialogsQuick.SelectServiceKey( service_types = HC.ALL_TAG_SERVICES, unallowed = [ self._service_key ] )
if service_key is None:
return
self._service_key = service_key
self._UpdateServiceKeyButtonLabel()
def _SetValue( self, importer: ClientMetadataMigrationImporters.SingleFileMetadataImporter ):
self._current_importer_class = type( importer )
self._change_type_button.setText( choice_tuple_label_lookup[ self._current_importer_class ] )
string_processor = importer.GetStringProcessor()
self._string_processor_button.SetValue( string_processor )
self._service_selection_panel.setVisible( False )
self._json_parsing_formula_panel.setVisible( False )
self._suffix_panel.setVisible( False )
if isinstance( importer, ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaTags ):
self._service_key = importer.GetServiceKey()
self._UpdateServiceKeyButtonLabel()
self._service_selection_panel.setVisible( True )
elif isinstance( importer, ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaURLs ):
pass
elif isinstance( importer, ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT ):
suffix = importer.GetSuffix()
self._suffix.setText( suffix )
self._suffix_panel.setVisible( True )
elif isinstance( importer, ClientMetadataMigrationImporters.SingleFileMetadataImporterJSON ):
suffix = importer.GetSuffix()
self._suffix.setText( suffix )
self._suffix_panel.setVisible( True )
self._json_parsing_formula = importer.GetJSONParsingFormula()
self._json_parsing_formula_panel.setVisible( True )
else:
raise Exception( 'Did not understand the new importer type!' )
def _UpdateServiceKeyButtonLabel( self ):
name = HG.client_controller.services_manager.GetName( self._service_key )
self._service_selection_button.setText( name )
def GetValue( self ) -> ClientMetadataMigrationImporters.SingleFileMetadataImporter:
importer = self._GetValue()
return importer
def convert_importer_to_pretty_string( importer: ClientMetadataMigrationImporters.SingleFileMetadataImporter ) -> str:
return importer.ToString()
class SingleFileMetadataImportersControl( ClientGUIListBoxes.AddEditDeleteListBox ):
def __init__( self, parent: QW.QWidget, importers: typing.Collection[ ClientMetadataMigrationImporters.SingleFileMetadataImporter ], allowed_importer_classes: list ):
ClientGUIListBoxes.AddEditDeleteListBox.__init__( self, parent, 5, convert_importer_to_pretty_string, self._AddImporter, self._EditImporter )
self._allowed_importer_classes = allowed_importer_classes
self.AddDatas( importers )
def _AddImporter( self ):
try:
importer_class = SelectClass( self, self._allowed_importer_classes )
except HydrusExceptions.CancelledException:
raise HydrusExceptions.VetoException()
string_processor = ClientStrings.StringProcessor()
importer = importer_class( string_processor )
with ClientGUITopLevelWindowsPanels.DialogEdit( self, 'edit metadata migration source' ) as dlg:
panel = EditSingleFileMetadataImporterPanel( self, importer, self._allowed_importer_classes )
dlg.SetPanel( panel )
if dlg.exec() == QW.QDialog.Accepted:
importer = panel.GetValue()
return importer
raise HydrusExceptions.VetoException()
def _EditImporter( self, importer: ClientMetadataMigrationImporters.SingleFileMetadataImporter ):
with ClientGUITopLevelWindowsPanels.DialogEdit( self, 'edit metadata migration source' ) as dlg:
panel = EditSingleFileMetadataImporterPanel( self, importer, self._allowed_importer_classes )
dlg.SetPanel( panel )
if dlg.exec() == QW.QDialog.Accepted:
edited_importer = panel.GetValue()
return edited_importer
raise HydrusExceptions.VetoException()

1
hydrus/client/gui/metadata/__init__.py Normal file
View File

@ -0,0 +1 @@

5
hydrus/client/gui/pages/ClientGUIManagement.py
View File

@ -58,6 +58,7 @@ from hydrus.client.importing.options import FileImportOptions
from hydrus.client.importing.options import PresentationImportOptions
from hydrus.client.media import ClientMedia
from hydrus.client.metadata import ClientTags
from hydrus.client.metadata import ClientMetadataMigration
MANAGEMENT_TYPE_DUMPER = 0
MANAGEMENT_TYPE_IMPORT_MULTIPLE_GALLERY = 1
@ -199,13 +200,13 @@ def CreateManagementControllerImportSimpleDownloader():
return management_controller
def CreateManagementControllerImportHDD( paths, file_import_options: FileImportOptions.FileImportOptions, paths_to_additional_service_keys_to_tags, delete_after_success ):
def CreateManagementControllerImportHDD( paths, file_import_options: FileImportOptions.FileImportOptions, metadata_routers: typing.Collection[ ClientMetadataMigration.SingleFileMetadataRouter ], paths_to_additional_service_keys_to_tags, delete_after_success ):
location_context = file_import_options.GetDestinationLocationContext()
management_controller = CreateManagementController( 'import', MANAGEMENT_TYPE_IMPORT_HDD, location_context = location_context )
hdd_import = ClientImportLocal.HDDImport( paths = paths, file_import_options = file_import_options, paths_to_additional_service_keys_to_tags = paths_to_additional_service_keys_to_tags, delete_after_success = delete_after_success )
hdd_import = ClientImportLocal.HDDImport( paths = paths, file_import_options = file_import_options, metadata_routers = metadata_routers, paths_to_additional_service_keys_to_tags = paths_to_additional_service_keys_to_tags, delete_after_success = delete_after_success )
management_controller.SetVariable( 'hdd_import', hdd_import )

8
hydrus/client/gui/pages/ClientGUIPages.py
View File

@ -558,6 +558,9 @@ class Page( QW.QWidget ):
old_panel = self._media_panel
self._media_panel = new_panel
# note focus isn't on the thumb panel but some innerwidget scroll gubbins
had_focus_before = ClientGUIFunctions.IsQtAncestor( QW.QApplication.focusWidget(), old_panel )
# this sets parent of new panel to self and sets parent of old panel to None
# rumao, it doesn't work if new_panel is already our child
self._management_media_split.replaceWidget( 1, new_panel )
@ -572,6 +575,11 @@ class Page( QW.QWidget ):
self._controller.pub( 'notify_new_pages_count' )
if had_focus_before:
ClientGUIFunctions.SetFocusLater( new_panel )
# if we try to kill a media page while a menu is open on it, we can enter program instability.
# so let's just put it off.
def clean_up_old_panel():

2
hydrus/client/gui/pages/ClientGUIResults.py
View File

@ -27,7 +27,6 @@ from hydrus.client.gui import ClientGUIDialogs
from hydrus.client.gui import ClientGUIDialogsManage
from hydrus.client.gui import ClientGUIDialogsQuick
from hydrus.client.gui import ClientGUIDuplicates
from hydrus.client.gui import ClientGUIExport
from hydrus.client.gui import ClientGUIFunctions
from hydrus.client.gui import ClientGUIMedia
from hydrus.client.gui import ClientGUIMediaActions
@ -40,6 +39,7 @@ from hydrus.client.gui import ClientGUITopLevelWindowsPanels
from hydrus.client.gui import QtPorting as QP
from hydrus.client.gui.canvas import ClientGUICanvas
from hydrus.client.gui.canvas import ClientGUICanvasFrame
from hydrus.client.gui.exporting import ClientGUIExport
from hydrus.client.gui.networking import ClientGUIHydrusNetwork
from hydrus.client.media import ClientMedia
from hydrus.client.metadata import ClientTags

25
hydrus/client/gui/search/ClientGUIACDropdown.py
View File

@ -1425,6 +1425,11 @@ class AutoCompleteDropdownTags( AutoCompleteDropdown ):
HG.client_controller.sub( self, 'NotifyNewServices', 'notify_new_services' )
def _BroadcastChoices( self, predicates, shift_down ):
raise NotImplementedError()
def _GetCurrentBroadcastTextPredicate( self ) -> typing.Optional[ ClientSearch.Predicate ]:
raise NotImplementedError()
@ -1446,6 +1451,11 @@ class AutoCompleteDropdownTags( AutoCompleteDropdown ):
raise NotImplementedError()
def _InitSearchResultsList( self ):
raise NotImplementedError()
def _LocationContextJustChanged( self, location_context: ClientLocation.LocationContext ):
self._RestoreTextCtrlFocus()
@ -1516,6 +1526,16 @@ class AutoCompleteDropdownTags( AutoCompleteDropdown ):
return True
def _ShouldTakeResponsibilityForEnter( self ):
raise NotImplementedError()
def _StartSearchResultsFetchJob( self, job_key ):
raise NotImplementedError()
def _TagContextJustChanged( self, tag_context: ClientSearch.TagContext ):
self._RestoreTextCtrlFocus()
@ -1551,6 +1571,11 @@ class AutoCompleteDropdownTags( AutoCompleteDropdown ):
return True
def _TakeResponsibilityForEnter( self, shift_down ):
raise NotImplementedError()
def NotifyNewServices( self ):
self._SetLocationContext( self._location_context_button.GetValue() )

1
hydrus/client/gui/search/ClientGUILocation.py
View File

@ -119,6 +119,7 @@ class EditMultipleLocationContextPanel( ClientGUIScrolledPanels.EditPanel ):
self._location_list.checkBoxListChanged.emit()
class LocationSearchContextButton( ClientGUICommon.BetterButton ):
locationChanged = QC.Signal( ClientLocation.LocationContext )

224
hydrus/client/importing/ClientImportLocal.py
View File

@ -1,6 +1,8 @@
import collections
import os
import threading
import time
import typing
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
@ -22,18 +24,36 @@ from hydrus.client.importing import ClientImporting
from hydrus.client.importing import ClientImportFileSeeds
from hydrus.client.importing.options import FileImportOptions
from hydrus.client.importing.options import TagImportOptions
from hydrus.client.metadata import ClientMetadataMigration
from hydrus.client.metadata import ClientMetadataMigrationExporters
from hydrus.client.metadata import ClientMetadataMigrationImporters
from hydrus.client.metadata import ClientTags
class HDDImport( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_HDD_IMPORT
SERIALISABLE_NAME = 'Local File Import'
SERIALISABLE_VERSION = 2
SERIALISABLE_VERSION = 3
def __init__( self, paths = None, file_import_options = None, paths_to_additional_service_keys_to_tags = None, delete_after_success = None ):
def __init__( self, paths = None, file_import_options = None, metadata_routers = None, paths_to_additional_service_keys_to_tags = None, delete_after_success = None ):
HydrusSerialisable.SerialisableBase.__init__( self )
if metadata_routers is None:
metadata_routers = []
if paths_to_additional_service_keys_to_tags is None:
paths_to_additional_service_keys_to_tags = collections.defaultdict( ClientTags.ServiceKeysToTags )
if delete_after_success is None:
delete_after_success = False
if paths is None:
self._file_seed_cache = None
@ -70,6 +90,8 @@ class HDDImport( HydrusSerialisable.SerialisableBase ):
self._file_seed_cache.AddFileSeeds( file_seeds )
self._metadata_routers = HydrusSerialisable.SerialisableList( metadata_routers )
self._file_import_options = file_import_options
self._delete_after_success = delete_after_success
@ -91,16 +113,18 @@ class HDDImport( HydrusSerialisable.SerialisableBase ):
serialisable_file_seed_cache = self._file_seed_cache.GetSerialisableTuple()
serialisable_options = self._file_import_options.GetSerialisableTuple()
serialisable_metadata_routers = self._metadata_routers.GetSerialisableTuple()
return ( serialisable_file_seed_cache, serialisable_options, self._delete_after_success, self._paused )
return ( serialisable_file_seed_cache, serialisable_options, serialisable_metadata_routers, self._delete_after_success, self._paused )
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_file_seed_cache, serialisable_options, self._delete_after_success, self._paused ) = serialisable_info
( serialisable_file_seed_cache, serialisable_options, serialisable_metadata_routers, self._delete_after_success, self._paused ) = serialisable_info
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_options )
self._metadata_routers = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_metadata_routers )
def _SerialisableChangeMade( self ):
@ -135,6 +159,19 @@ class HDDImport( HydrusSerialisable.SerialisableBase ):
return ( 2, new_serialisable_info )
if version == 2:
( serialisable_file_seed_cache, serialisable_options, delete_after_success, paused ) = old_serialisable_info
metadata_routers = HydrusSerialisable.SerialisableList()
serialisable_metadata_routers = metadata_routers.GetSerialisableTuple()
new_serialisable_info = ( serialisable_file_seed_cache, serialisable_options, serialisable_metadata_routers, delete_after_success, paused )
return ( 3, new_serialisable_info )
def _WorkOnFiles( self ):
@ -164,6 +201,26 @@ class HDDImport( HydrusSerialisable.SerialisableBase ):
if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:
if len( self._metadata_routers ) > 0:
hash = file_seed.GetHash()
media_result = HG.client_controller.Read( 'media_result', hash )
for router in self._metadata_routers:
try:
router.Work( media_result, file_seed.file_seed_data )
except Exception as e:
HydrusData.ShowText( 'Trying to run metadata routing on the file "{}" threw an error!'.format( file_seed.file_seed_data ) )
HydrusData.ShowException( e )
real_presentation_import_options = FileImportOptions.GetRealPresentationImportOptions( self._file_import_options, FileImportOptions.IMPORT_TYPE_LOUD )
if file_seed.ShouldPresent( real_presentation_import_options ):
@ -179,25 +236,32 @@ class HDDImport( HydrusSerialisable.SerialisableBase ):
except Exception as e:
HydrusData.ShowText( 'While attempting to delete ' + path + ', the following error occurred:' )
HydrusData.ShowText( 'While attempting to delete {}, the following error occurred:'.format( path ) )
HydrusData.ShowException( e )
txt_path = path + '.txt'
possible_sidecar_paths = set()
if os.path.exists( txt_path ):
for router in self._metadata_routers:
try:
ClientPaths.DeletePath( txt_path )
except Exception as e:
HydrusData.ShowText( 'While attempting to delete ' + txt_path + ', the following error occurred:' )
HydrusData.ShowException( e )
possible_sidecar_paths.update( router.GetPossibleImporterSidecarPaths( path ) )
for possible_sidecar_path in possible_sidecar_paths:
if os.path.exists( possible_sidecar_path ):
try:
ClientPaths.DeletePath( possible_sidecar_path )
except Exception as e:
HydrusData.ShowText( 'While attempting to delete {}, the following error occurred:'.format( possible_sidecar_path ) )
HydrusData.ShowException( e )
with self._lock:
@ -390,9 +454,24 @@ class ImportFolder( HydrusSerialisable.SerialisableBaseNamed ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_IMPORT_FOLDER
SERIALISABLE_NAME = 'Import Folder'
SERIALISABLE_VERSION = 7
SERIALISABLE_VERSION = 8
def __init__( self, name, path = '', file_import_options = None, tag_import_options = None, tag_service_keys_to_filename_tagging_options = None, actions = None, action_locations = None, period = 3600, check_regularly = True, show_working_popup = True, publish_files_to_popup_button = True, publish_files_to_page = False ):
def __init__(
self,
name,
path = '',
file_import_options = None,
tag_import_options = None,
metadata_routers: typing.Optional[ typing.Collection[ ClientMetadataMigration.SingleFileMetadataRouter ] ] = None,
tag_service_keys_to_filename_tagging_options = None,
actions = None,
action_locations = None,
period = 3600,
check_regularly = True,
show_working_popup = True,
publish_files_to_popup_button = True,
publish_files_to_page = False
):
if file_import_options is None:
@ -405,6 +484,13 @@ class ImportFolder( HydrusSerialisable.SerialisableBaseNamed ):
tag_import_options = TagImportOptions.TagImportOptions()
if metadata_routers is None:
metadata_routers = []
metadata_routers = HydrusSerialisable.SerialisableList( metadata_routers )
if tag_service_keys_to_filename_tagging_options is None:
tag_service_keys_to_filename_tagging_options = {}
@ -430,6 +516,7 @@ class ImportFolder( HydrusSerialisable.SerialisableBaseNamed ):
self._path = path
self._file_import_options = file_import_options
self._tag_import_options = tag_import_options
self._metadata_routers = metadata_routers
self._tag_service_keys_to_filename_tagging_options = tag_service_keys_to_filename_tagging_options
self._actions = actions
self._action_locations = action_locations
@ -472,11 +559,19 @@ class ImportFolder( HydrusSerialisable.SerialisableBaseNamed ):
ClientPaths.DeletePath( path )
txt_path = path + '.txt'
possible_sidecar_paths = set()
if os.path.exists( txt_path ):
for router in self._metadata_routers:
ClientPaths.DeletePath( txt_path )
possible_sidecar_paths.update( router.GetPossibleImporterSidecarPaths( path ) )
for possible_sidecar_path in possible_sidecar_paths:
if os.path.exists( possible_sidecar_path ):
ClientPaths.DeletePath( possible_sidecar_path )
self._file_seed_cache.RemoveFileSeeds( ( file_seed, ) )
@ -613,6 +708,7 @@ class ImportFolder( HydrusSerialisable.SerialisableBaseNamed ):
serialisable_file_import_options = self._file_import_options.GetSerialisableTuple()
serialisable_tag_import_options = self._tag_import_options.GetSerialisableTuple()
serialisable_metadata_routers = self._metadata_routers.GetSerialisableTuple()
serialisable_tag_service_keys_to_filename_tagging_options = [ ( service_key.hex(), filename_tagging_options.GetSerialisableTuple() ) for ( service_key, filename_tagging_options ) in list(self._tag_service_keys_to_filename_tagging_options.items()) ]
serialisable_file_seed_cache = self._file_seed_cache.GetSerialisableTuple()
@ -620,7 +716,7 @@ class ImportFolder( HydrusSerialisable.SerialisableBaseNamed ):
action_pairs = list(self._actions.items())
action_location_pairs = list(self._action_locations.items())
return ( self._path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, self._period, self._check_regularly, serialisable_file_seed_cache, self._last_checked, self._paused, self._check_now, self._show_working_popup, self._publish_files_to_popup_button, self._publish_files_to_page )
return ( self._path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_metadata_routers, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, self._period, self._check_regularly, serialisable_file_seed_cache, self._last_checked, self._paused, self._check_now, self._show_working_popup, self._publish_files_to_popup_button, self._publish_files_to_page )
def _ImportFiles( self, job_key ):
@ -678,23 +774,40 @@ class ImportFolder( HydrusSerialisable.SerialisableBaseNamed ):
hash = file_seed.GetHash()
if self._tag_import_options.HasAdditionalTags():
if self._tag_import_options.HasAdditionalTags() or len( self._metadata_routers ) > 0:
media_result = HG.client_controller.Read( 'media_result', hash )
downloaded_tags = []
service_keys_to_content_updates = self._tag_import_options.GetServiceKeysToContentUpdates( file_seed.status, media_result, downloaded_tags ) # additional tags
if len( service_keys_to_content_updates ) > 0:
if self._tag_import_options.HasAdditionalTags():
HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
downloaded_tags = []
service_keys_to_content_updates = self._tag_import_options.GetServiceKeysToContentUpdates( file_seed.status, media_result, downloaded_tags ) # additional tags
if len( service_keys_to_content_updates ) > 0:
HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
for metadata_router in self._metadata_routers:
try:
metadata_router.Work( media_result, path )
except Exception as e:
HydrusData.ShowText( 'Trying to run metadata routing in the import folder "' + self._name + '" threw an error!' )
HydrusData.ShowException( e )
service_keys_to_tags = ClientTags.ServiceKeysToTags()
for ( tag_service_key, filename_tagging_options ) in list(self._tag_service_keys_to_filename_tagging_options.items()):
for ( tag_service_key, filename_tagging_options ) in self._tag_service_keys_to_filename_tagging_options.items():
if not HG.client_controller.services_manager.ServiceExists( tag_service_key ):
@ -770,13 +883,14 @@ class ImportFolder( HydrusSerialisable.SerialisableBaseNamed ):
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( self._path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, self._period, self._check_regularly, serialisable_file_seed_cache, self._last_checked, self._paused, self._check_now, self._show_working_popup, self._publish_files_to_popup_button, self._publish_files_to_page ) = serialisable_info
( self._path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_metadata_routers, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, self._period, self._check_regularly, serialisable_file_seed_cache, self._last_checked, self._paused, self._check_now, self._show_working_popup, self._publish_files_to_popup_button, self._publish_files_to_page ) = serialisable_info
self._actions = dict( action_pairs )
self._action_locations = dict( action_location_pairs )
self._file_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_import_options )
self._tag_import_options = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_tag_import_options )
self._metadata_routers = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_metadata_routers )
self._tag_service_keys_to_filename_tagging_options = dict( [ ( bytes.fromhex( encoded_service_key ), HydrusSerialisable.CreateFromSerialisableTuple( serialisable_filename_tagging_options ) ) for ( encoded_service_key, serialisable_filename_tagging_options ) in serialisable_tag_service_keys_to_filename_tagging_options ] )
self._file_seed_cache = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_file_seed_cache )
@ -873,6 +987,44 @@ class ImportFolder( HydrusSerialisable.SerialisableBaseNamed ):
return ( 7, new_serialisable_info )
if version == 7:
( path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, check_regularly, serialisable_file_seed_cache, last_checked, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page ) = old_serialisable_info
tag_service_keys_to_filename_tagging_options = dict( [ ( bytes.fromhex( encoded_service_key ), HydrusSerialisable.CreateFromSerialisableTuple( serialisable_filename_tagging_options ) ) for ( encoded_service_key, serialisable_filename_tagging_options ) in serialisable_tag_service_keys_to_filename_tagging_options ] )
metadata_routers = HydrusSerialisable.SerialisableList()
try:
for ( service_key, filename_tagging_options ) in tag_service_keys_to_filename_tagging_options.items():
# beardy access here, but this is once off
if hasattr( filename_tagging_options, '_load_from_neighbouring_txt_files' ) and filename_tagging_options._load_from_neighbouring_txt_files:
importers = [ ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT() ]
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags( service_key = service_key )
metadata_router = ClientMetadataMigration.SingleFileMetadataRouter( importers = importers, exporter = exporter )
metadata_routers.append( metadata_router )
except Exception as e:
HydrusData.Print( 'Failed to update import folder with new metadata routers.' )
HydrusData.PrintException( e )
serialisable_metadata_routers = metadata_routers.GetSerialisableTuple()
new_serialisable_info = ( path, serialisable_file_import_options, serialisable_tag_import_options, serialisable_metadata_routers, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, check_regularly, serialisable_file_seed_cache, last_checked, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page )
return ( 8, new_serialisable_info )
def CheckNow( self ):
@ -970,6 +1122,11 @@ class ImportFolder( HydrusSerialisable.SerialisableBaseNamed ):
return self._file_seed_cache
def GetMetadataRouters( self ):
return list( self._metadata_routers )
def Paused( self ):
return self._paused
@ -995,6 +1152,11 @@ class ImportFolder( HydrusSerialisable.SerialisableBaseNamed ):
self._file_seed_cache = file_seed_cache
def SetMetadataRouters( self, metadata_routers: typing.Collection[ ClientMetadataMigration.SingleFileMetadataRouter ] ):
self._metadata_routers = HydrusSerialisable.SerialisableList( metadata_routers )
def SetTuple( self, name, path, file_import_options, tag_import_options, tag_service_keys_to_filename_tagging_options, actions, action_locations, period, check_regularly, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page ):
if path != self._path:

35
hydrus/client/importing/options/TagImportOptions.py
View File

@ -11,9 +11,9 @@ from hydrus.core import HydrusTags
from hydrus.client import ClientConstants as CC
from hydrus.client import ClientData
from hydrus.client.exporting import ClientExportingMetadata
from hydrus.client.importing.options import ClientImportOptions
from hydrus.client.media import ClientMediaResult
from hydrus.client.metadata import ClientMetadataMigrationImporters
from hydrus.client.metadata import ClientTags
class FilenameTaggingOptions( HydrusSerialisable.SerialisableBase ):
@ -28,6 +28,8 @@ class FilenameTaggingOptions( HydrusSerialisable.SerialisableBase ):
self._tags_for_all = set()
# Note we are leaving this here for a bit, even though it is no longer used, to leave a window so ImportFolder can rip existing values
# it can be nuked in due time
self._load_from_neighbouring_txt_files = False
self._add_filename = ( False, 'filename' )
@ -103,32 +105,6 @@ class FilenameTaggingOptions( HydrusSerialisable.SerialisableBase ):
tags.update( self._tags_for_all )
if self._load_from_neighbouring_txt_files:
# TODO: this needs more work, making an actual Router object with an Exporter, grinding things towards flexible conversion with different types and actually firing off content updates vs 'get example tags for UI'
# I'm pretty sure we could also make a 'FilenameImporter' and pipe all the following gubbins into the same system lad
importer = ClientExportingMetadata.SingleFileMetadataImporterExporterTXT()
try:
txt_tags = importer.Import( path )
if True in ( len( txt_tag ) > 1024 for txt_tag in txt_tags ):
raise Exception( 'Tags were too long--I think this was not a regular text file!' )
tags.update( txt_tags )
except Exception as e:
HydrusData.ShowText( 'Problem getting tags from a txt sidecar! {}'.format( e ) )
tags.add( '___had problem parsing .txt file' )
( base, filename ) = os.path.split( path )
( filename, any_ext_gumpf ) = os.path.splitext( filename )
@ -253,17 +229,16 @@ class FilenameTaggingOptions( HydrusSerialisable.SerialisableBase ):
return tags
def SimpleSetTuple( self, tags_for_all, load_from_neighbouring_txt_files, add_filename, directories_dict ):
def SimpleSetTuple( self, tags_for_all, add_filename, directories_dict ):
self._tags_for_all = tags_for_all
self._load_from_neighbouring_txt_files = load_from_neighbouring_txt_files
self._add_filename = add_filename
self._directories_dict = directories_dict
def SimpleToTuple( self ):
return ( self._tags_for_all, self._load_from_neighbouring_txt_files, self._add_filename, self._directories_dict )
return ( self._tags_for_all, self._add_filename, self._directories_dict )
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_FILENAME_TAGGING_OPTIONS ] = FilenameTaggingOptions

206
hydrus/client/metadata/ClientMetadataMigration.py Normal file
View File

@ -0,0 +1,206 @@
import typing
from hydrus.core import HydrusSerialisable
from hydrus.core import HydrusTags
from hydrus.client import ClientStrings
from hydrus.client.media import ClientMediaResult
from hydrus.client.metadata import ClientMetadataMigrationExporters
from hydrus.client.metadata import ClientMetadataMigrationImporters
class SingleFileMetadataRouter( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_ROUTER
SERIALISABLE_NAME = 'Metadata Single File Router'
SERIALISABLE_VERSION = 2
def __init__(
self,
importers: typing.Optional[ typing.Collection[ ClientMetadataMigrationImporters.SingleFileMetadataImporter ] ] = None,
string_processor: typing.Optional[ ClientStrings.StringProcessor ] = None,
exporter: typing.Optional[ ClientMetadataMigrationExporters.SingleFileMetadataExporter ] = None
):
if importers is None:
importers = []
if string_processor is None:
string_processor = ClientStrings.StringProcessor()
if exporter is None:
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT()
HydrusSerialisable.SerialisableBase.__init__( self )
self._importers = HydrusSerialisable.SerialisableList( importers )
self._string_processor = string_processor
self._exporter = exporter
def __str__( self ):
return self.ToString()
def _GetSerialisableInfo( self ):
serialisable_importers = self._importers.GetSerialisableTuple()
serialisable_string_processor = self._string_processor.GetSerialisableTuple()
serialisable_exporter = self._exporter.GetSerialisableTuple()
return ( serialisable_importers, serialisable_string_processor, serialisable_exporter )
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_importers, serialisable_string_processor, serialisable_exporter ) = serialisable_info
self._importers = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_importers )
self._string_processor = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_string_processor )
self._exporter = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_exporter )
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
if version == 1:
# in this version, we are moving from importer/exporter combined classes to separate. the importer/exporters are becoming importers, so importer/exporters in export slot need to be remade
( serialisable_importers, serialisable_string_processor, serialisable_exporter ) = old_serialisable_info
actually_an_importer = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_exporter )
if isinstance( actually_an_importer, ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT ):
suffix = actually_an_importer.GetSuffix()
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT( suffix )
elif isinstance( actually_an_importer, ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaTags ):
service_key = actually_an_importer.GetServiceKey()
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags( service_key )
else:
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT()
fixed_serialisable_exporter = exporter.GetSerialisableTuple()
new_serialisable_info = ( serialisable_importers, serialisable_string_processor, fixed_serialisable_exporter )
return ( 2, new_serialisable_info )
def GetExporter( self ) -> ClientMetadataMigrationExporters.SingleFileMetadataExporter:
return self._exporter
def GetImporters( self ) -> typing.List[ ClientMetadataMigrationImporters.SingleFileMetadataImporter ]:
return list( self._importers )
def GetPossibleImporterSidecarPaths( self, path ):
sidecar_importers = [ importer for importer in self._importers if isinstance( importer, ClientMetadataMigrationImporters.SingleFileMetadataImporterSidecar ) ]
possible_sidecar_paths = { importer.GetExpectedSidecarPath( path ) for importer in sidecar_importers }
return possible_sidecar_paths
def GetStringProcessor( self ) -> ClientStrings.StringProcessor:
return self._string_processor
def ToString( self, pretty = False ) -> str:
if len( self._importers ) > 0:
source_text = ', '.join( ( importer.ToString() for importer in self._importers ) )
else:
source_text = 'nothing'
if self._string_processor.MakesChanges():
full_munge_text = ', applying {}'.format( self._string_processor.ToString() )
else:
full_munge_text = ''
dest_text = self._exporter.ToString()
if pretty:
header = ''
else:
header = 'Single File Metadata Router: '
return '{}Taking {}{}, sending {}.'.format( header, source_text, full_munge_text, dest_text )
def Work( self, media_result: ClientMediaResult.MediaResult, file_path: str ):
rows = set()
for importer in self._importers:
if isinstance( importer, ClientMetadataMigrationImporters.SingleFileMetadataImporterSidecar ):
rows.update( importer.Import( file_path ) )
elif isinstance( importer, ClientMetadataMigrationImporters.SingleFileMetadataImporterMedia ):
rows.update( importer.Import( media_result ) )
else:
raise Exception( 'Problem with importer object!' )
rows = sorted( rows, key = HydrusTags.ConvertTagToSortable )
rows = self._string_processor.ProcessStrings( starting_strings = rows )
if len( rows ) == 0:
return
if isinstance( self._exporter, ClientMetadataMigrationExporters.SingleFileMetadataExporterSidecar ):
self._exporter.Export( file_path, rows )
elif isinstance( self._exporter, ClientMetadataMigrationExporters.SingleFileMetadataExporterMedia ):
self._exporter.Export( media_result.GetHash(), rows )
else:
raise Exception( 'Problem with exporter object!' )
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_ROUTER ] = SingleFileMetadataRouter

60
hydrus/client/metadata/ClientMetadataMigrationCore.py Normal file
View File

@ -0,0 +1,60 @@
def GetSidecarPath( actual_file_path: str, suffix: str, file_extension: str ):
path_components = [ actual_file_path ]
if suffix != '':
path_components.append( suffix )
path_components.append( file_extension )
return '.'.join( path_components )
class ImporterExporterNode( object ):
def __str__( self ):
return self.ToString()
def GetExampleStrings( self ):
examples = [
'blue eyes',
'blonde hair',
'skirt',
'character:jane smith',
'series:jane smith adventures',
'creator:some guy',
'https://example.com/gallery/index.php?post=123456&page=show',
'https://cdn3.expl.com/files/file_id?id=123456&token=0123456789abcdef'
]
return examples
def ToString( self ) -> str:
raise NotImplementedError()
class SidecarNode( object ):
def __init__( self, suffix: str ):
self._suffix = suffix
def GetSuffix( self ) -> str:
return self._suffix
def SetSuffix( self, suffix: str ):
self._suffix = suffix

404
hydrus/client/metadata/ClientMetadataMigrationExporters.py Normal file
View File

@ -0,0 +1,404 @@
import json
import os
import typing
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusGlobals as HG
from hydrus.core import HydrusSerialisable
from hydrus.client import ClientConstants as CC
from hydrus.client.metadata import ClientMetadataMigrationCore
class SingleFileMetadataExporter( ClientMetadataMigrationCore.ImporterExporterNode ):
def Export( self, *args, **kwargs ):
raise NotImplementedError()
def ToString( self ) -> str:
raise NotImplementedError()
class SingleFileMetadataExporterMedia( SingleFileMetadataExporter ):
def Export( self, hash: bytes, rows: typing.Collection[ str ] ):
raise NotImplementedError()
def ToString( self ) -> str:
raise NotImplementedError()
class SingleFileMetadataExporterSidecar( SingleFileMetadataExporter, ClientMetadataMigrationCore.SidecarNode ):
def __init__( self, suffix: str ):
ClientMetadataMigrationCore.SidecarNode.__init__( self, suffix )
SingleFileMetadataExporter.__init__( self )
def Export( self, actual_file_path: str, rows: typing.Collection[ str ] ):
raise NotImplementedError()
def ToString( self ) -> str:
raise NotImplementedError()
class SingleFileMetadataExporterMediaTags( HydrusSerialisable.SerialisableBase, SingleFileMetadataExporterMedia ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_EXPORTER_MEDIA_TAGS
SERIALISABLE_NAME = 'Metadata Single File Exporter Media Tags'
SERIALISABLE_VERSION = 1
def __init__( self, service_key = None ):
HydrusSerialisable.SerialisableBase.__init__( self )
SingleFileMetadataExporterMedia.__init__( self )
if service_key is None:
service_key = CC.DEFAULT_LOCAL_TAG_SERVICE_KEY
self._service_key = service_key
def _GetSerialisableInfo( self ):
return self._service_key.hex()
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
serialisable_service_key = serialisable_info
self._service_key = bytes.fromhex( serialisable_service_key )
def GetExampleStrings( self ):
examples = [
'blue eyes',
'blonde hair',
'skirt',
'character:jane smith',
'series:jane smith adventures',
'creator:some guy'
]
return examples
def GetServiceKey( self ) -> bytes:
return self._service_key
def Export( self, hash: bytes, rows: typing.Collection[ str ] ):
if len( rows ) == 0:
return
if HG.client_controller.services_manager.GetServiceType( self._service_key ) == HC.LOCAL_TAG:
add_content_action = HC.CONTENT_UPDATE_ADD
else:
add_content_action = HC.CONTENT_UPDATE_PEND
hashes = { hash }
content_updates = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, add_content_action, ( tag, hashes ) ) for tag in rows ]
HG.client_controller.WriteSynchronous( 'content_updates', { self._service_key : content_updates } )
def SetServiceKey( self, service_key: bytes ):
self._service_key = service_key
def ToString( self ) -> str:
try:
name = HG.client_controller.services_manager.GetName( self._service_key )
except:
name = 'unknown service'
return 'tags to media, on "{}"'.format( name )
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_EXPORTER_MEDIA_TAGS ] = SingleFileMetadataExporterMediaTags
class SingleFileMetadataExporterMediaURLs( HydrusSerialisable.SerialisableBase, SingleFileMetadataExporterMedia ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_EXPORTER_MEDIA_URLS
SERIALISABLE_NAME = 'Metadata Single File Exporter Media URLs'
SERIALISABLE_VERSION = 1
def __init__( self ):
HydrusSerialisable.SerialisableBase.__init__( self )
SingleFileMetadataExporterMedia.__init__( self )
def _GetSerialisableInfo( self ):
return list()
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
gumpf = serialisable_info
def Export( self, hash: bytes, rows: typing.Collection[ str ] ):
if len( rows ) == 0:
return
urls = []
for row in rows:
try:
url = HG.client_controller.network_engine.domain_manager.NormaliseURL( row )
urls.append( url )
except HydrusExceptions.URLClassException:
continue
except:
continue
hashes = { hash }
content_updates = [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( urls, hashes ) ) ]
HG.client_controller.WriteSynchronous( 'content_updates', { CC.COMBINED_LOCAL_FILE_SERVICE_KEY : content_updates } )
def GetExampleStrings( self ):
examples = [
'https://example.com/gallery/index.php?post=123456&page=show',
'https://cdn3.expl.com/files/file_id?id=123456&token=0123456789abcdef'
]
return examples
def ToString( self ) -> str:
return 'urls to media'
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_EXPORTER_MEDIA_URLS ] = SingleFileMetadataExporterMediaURLs
class SingleFileMetadataExporterJSON( HydrusSerialisable.SerialisableBase, SingleFileMetadataExporterSidecar ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_EXPORTER_JSON
SERIALISABLE_NAME = 'Metadata Single File Exporter JSON'
SERIALISABLE_VERSION = 1
def __init__( self, suffix = None, nested_object_names = None ):
if suffix is None:
suffix = ''
HydrusSerialisable.SerialisableBase.__init__( self )
SingleFileMetadataExporterSidecar.__init__( self, suffix )
if nested_object_names is None:
nested_object_names = []
self._nested_object_names = nested_object_names
def _GetSerialisableInfo( self ):
return ( self._suffix, self._nested_object_names )
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( self._suffix, self._nested_object_names ) = serialisable_info
def Export( self, actual_file_path: str, rows: typing.Collection[ str ] ):
if len( rows ) == 0:
return
path = ClientMetadataMigrationCore.GetSidecarPath( actual_file_path, self._suffix, 'json' )
if len( self._nested_object_names ) > 0:
if os.path.exists( path ):
with open( path, 'r', encoding = 'utf-8') as f:
existing_raw_json = f.read()
try:
json_dict = json.loads( existing_raw_json )
if len( self._nested_object_names ) > 0 and not isinstance( json_dict, dict ):
raise Exception( 'The existing JSON file was not a JSON Object!' )
except Exception as e:
# TODO: we probably want custom importer/exporter exceptions here
raise Exception( 'Could not read the existing JSON at {}!{}{}'.format( path, os.linesep, e ) )
else:
json_dict = dict()
node = json_dict
for ( i, name ) in enumerate( self._nested_object_names ):
if i == len( self._nested_object_names ) - 1:
node[ name ] = list( rows )
else:
if name not in node:
node[ name ] = dict()
node = node[ name ]
json_to_write = json_dict
else:
json_to_write = list( rows )
raw_json_to_write = json.dumps( json_to_write )
with open( path, 'w', encoding = 'utf-8' ) as f:
f.write( raw_json_to_write )
def GetNestedObjectNames( self ) -> typing.List[ str ]:
return list( self._nested_object_names )
def SetNestedObjectNames( self, nested_object_names: typing.List[ str ] ):
self._nested_object_names = list( nested_object_names )
def ToString( self ) -> str:
suffix_s = '' if self._suffix == '' else '.{}'.format( self._suffix )
return 'to {}.json sidecar ({})'.format( suffix_s, '>'.join( self._nested_object_names ) )
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_EXPORTER_JSON ] = SingleFileMetadataExporterJSON
class SingleFileMetadataExporterTXT( HydrusSerialisable.SerialisableBase, SingleFileMetadataExporterSidecar ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_EXPORTER_TXT
SERIALISABLE_NAME = 'Metadata Single File Exporter TXT'
SERIALISABLE_VERSION = 1
def __init__( self, suffix = None ):
if suffix is None:
suffix = ''
HydrusSerialisable.SerialisableBase.__init__( self )
SingleFileMetadataExporterSidecar.__init__( self, suffix )
def _GetSerialisableInfo( self ):
return self._suffix
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
self._suffix = serialisable_info
def Export( self, actual_file_path: str, rows: typing.Collection[ str ] ):
if len( rows ) == 0:
return
path = ClientMetadataMigrationCore.GetSidecarPath( actual_file_path, self._suffix, 'txt' )
with open( path, 'w', encoding = 'utf-8' ) as f:
f.write( '\n'.join( rows ) )
def ToString( self ) -> str:
suffix_s = '' if self._suffix == '' else '.{}'.format( self._suffix )
return 'to {}.txt sidecar'.format( suffix_s )
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_EXPORTER_TXT ] = SingleFileMetadataExporterTXT

513
hydrus/client/metadata/ClientMetadataMigrationImporters.py Normal file
View File

@ -0,0 +1,513 @@
import os
import typing
from hydrus.core import HydrusGlobals as HG
from hydrus.core import HydrusSerialisable
from hydrus.core import HydrusText
from hydrus.client import ClientConstants as CC
from hydrus.client import ClientParsing
from hydrus.client import ClientStrings
from hydrus.client.media import ClientMediaResult
from hydrus.client.metadata import ClientMetadataMigrationCore
from hydrus.client.metadata import ClientTags
# TODO: All importers should probably have a string processor
class SingleFileMetadataImporter( ClientMetadataMigrationCore.ImporterExporterNode ):
def __init__( self, string_processor: ClientStrings.StringProcessor ):
self._string_processor = string_processor
def GetStringProcessor( self ) -> ClientStrings.StringProcessor:
return self._string_processor
def Import( self, *args, **kwargs ):
raise NotImplementedError()
def ToString( self ) -> str:
raise NotImplementedError()
class SingleFileMetadataImporterMedia( SingleFileMetadataImporter ):
def Import( self, media_result: ClientMediaResult.MediaResult ):
raise NotImplementedError()
def ToString( self ) -> str:
raise NotImplementedError()
class SingleFileMetadataImporterSidecar( SingleFileMetadataImporter, ClientMetadataMigrationCore.SidecarNode ):
def __init__( self, string_processor: ClientStrings.StringProcessor, suffix: str ):
ClientMetadataMigrationCore.SidecarNode.__init__( self, suffix )
SingleFileMetadataImporter.__init__( self, string_processor )
def GetExpectedSidecarPath( self, path: str ):
raise NotImplementedError()
def Import( self, actual_file_path: str ):
raise NotImplementedError()
def ToString( self ) -> str:
raise NotImplementedError()
class SingleFileMetadataImporterMediaTags( HydrusSerialisable.SerialisableBase, SingleFileMetadataImporterMedia ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_MEDIA_TAGS
SERIALISABLE_NAME = 'Metadata Single File Importer Media Tags'
SERIALISABLE_VERSION = 2
def __init__( self, string_processor = None, service_key = None ):
if string_processor is None:
string_processor = ClientStrings.StringProcessor()
HydrusSerialisable.SerialisableBase.__init__( self )
SingleFileMetadataImporterMedia.__init__( self, string_processor )
if service_key is None:
service_key = CC.COMBINED_TAG_SERVICE_KEY
self._service_key = service_key
def _GetSerialisableInfo( self ):
serialisable_string_processor = self._string_processor.GetSerialisableTuple()
serialisable_service_key = self._service_key.hex()
return ( serialisable_string_processor, serialisable_service_key )
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_string_processor, serialisable_service_key ) = serialisable_info
self._string_processor = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_string_processor )
self._service_key = bytes.fromhex( serialisable_service_key )
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
if version == 1:
serialisable_service_key = old_serialisable_info
string_processor = ClientStrings.StringProcessor()
serialisable_string_processor = string_processor.GetSerialisableTuple()
new_serialisable_info = ( serialisable_string_processor, serialisable_service_key )
return ( 2, new_serialisable_info )
def GetExampleStrings( self ):
examples = [
'blue eyes',
'blonde hair',
'skirt',
'character:jane smith',
'series:jane smith adventures',
'creator:some guy'
]
return examples
def GetServiceKey( self ) -> bytes:
return self._service_key
def Import( self, media_result: ClientMediaResult.MediaResult ):
tags = media_result.GetTagsManager().GetCurrent( self._service_key, ClientTags.TAG_DISPLAY_STORAGE )
if self._string_processor.MakesChanges():
tags = self._string_processor.ProcessStrings( tags )
return tags
def SetServiceKey( self, service_key: bytes ):
self._service_key = service_key
def ToString( self ) -> str:
try:
name = HG.client_controller.services_manager.GetName( self._service_key )
except:
name = 'unknown service'
if self._string_processor.MakesChanges():
full_munge_text = ', applying {}'.format( self._string_processor.ToString() )
else:
full_munge_text = ''
return '"{}" tags from media{}'.format( name, full_munge_text )
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_MEDIA_TAGS ] = SingleFileMetadataImporterMediaTags
class SingleFileMetadataImporterMediaURLs( HydrusSerialisable.SerialisableBase, SingleFileMetadataImporterMedia ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_MEDIA_URLS
SERIALISABLE_NAME = 'Metadata Single File Importer Media URLs'
SERIALISABLE_VERSION = 2
def __init__( self, string_processor = None ):
if string_processor is None:
string_processor = ClientStrings.StringProcessor()
HydrusSerialisable.SerialisableBase.__init__( self )
SingleFileMetadataImporterMedia.__init__( self, string_processor )
def _GetSerialisableInfo( self ):
serialisable_string_processor = self._string_processor.GetSerialisableTuple()
return serialisable_string_processor
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
serialisable_string_processor = serialisable_info
self._string_processor = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_string_processor )
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
if version == 1:
gumpf = old_serialisable_info
string_processor = ClientStrings.StringProcessor()
serialisable_string_processor = string_processor.GetSerialisableTuple()
new_serialisable_info = serialisable_string_processor
return ( 2, new_serialisable_info )
def GetExampleStrings( self ):
examples = [
'https://example.com/gallery/index.php?post=123456&page=show',
'https://cdn3.expl.com/files/file_id?id=123456&token=0123456789abcdef'
]
return examples
def Import( self, media_result: ClientMediaResult.MediaResult ):
urls = media_result.GetLocationsManager().GetURLs()
if self._string_processor.MakesChanges():
urls = self._string_processor.ProcessStrings( urls )
return urls
def ToString( self ) -> str:
if self._string_processor.MakesChanges():
full_munge_text = ', applying {}'.format( self._string_processor.ToString() )
else:
full_munge_text = ''
return 'urls from media{}'.format( full_munge_text )
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_MEDIA_URLS ] = SingleFileMetadataImporterMediaURLs
class SingleFileMetadataImporterJSON( HydrusSerialisable.SerialisableBase, SingleFileMetadataImporterSidecar ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_JSON
SERIALISABLE_NAME = 'Metadata Single File Importer JSON'
SERIALISABLE_VERSION = 2
def __init__( self, string_processor = None, suffix = None, json_parsing_formula = None ):
if suffix is None:
suffix = ''
if string_processor is None:
string_processor = ClientStrings.StringProcessor()
HydrusSerialisable.SerialisableBase.__init__( self )
SingleFileMetadataImporterSidecar.__init__( self, string_processor, suffix )
if json_parsing_formula is None:
parse_rules = [ ( ClientParsing.JSON_PARSE_RULE_TYPE_ALL_ITEMS, None ) ]
json_parsing_formula = ClientParsing.ParseFormulaJSON( parse_rules = parse_rules, content_to_fetch = ClientParsing.JSON_CONTENT_STRING )
self._json_parsing_formula = json_parsing_formula
def _GetSerialisableInfo( self ):
serialisable_string_processor = self._string_processor.GetSerialisableTuple()
serialisable_json_parsing_formula = self._json_parsing_formula.GetSerialisableTuple()
return ( serialisable_string_processor, self._suffix, serialisable_json_parsing_formula )
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_string_processor, self._suffix, serialisable_json_parsing_formula ) = serialisable_info
self._string_processor = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_string_processor )
self._json_parsing_formula = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_json_parsing_formula )
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
if version == 1:
( suffix, serialisable_json_parsing_formula ) = old_serialisable_info
string_processor = ClientStrings.StringProcessor()
serialisable_string_processor = string_processor.GetSerialisableTuple()
new_serialisable_info = ( serialisable_string_processor, suffix, serialisable_json_parsing_formula )
return ( 2, new_serialisable_info )
def GetExpectedSidecarPath( self, actual_file_path: str ):
return ClientMetadataMigrationCore.GetSidecarPath( actual_file_path, self._suffix, 'json' )
def GetJSONParsingFormula( self ) -> ClientParsing.ParseFormulaJSON:
return self._json_parsing_formula
def Import( self, actual_file_path: str ) -> typing.Collection[ str ]:
path = self.GetExpectedSidecarPath( actual_file_path )
if not os.path.exists( path ):
return []
try:
with open( path, 'r', encoding = 'utf-8' ) as f:
read_raw_json = f.read()
except Exception as e:
raise Exception( 'Could not import from {}: {}'.format( path, str( e ) ) )
parsing_context = {}
collapse_newlines = False
rows = self._json_parsing_formula.Parse( parsing_context, read_raw_json, collapse_newlines )
if self._string_processor.MakesChanges():
rows = self._string_processor.ProcessStrings( rows )
return rows
def SetJSONParsingFormula( self, json_parsing_formula: ClientParsing.ParseFormulaJSON ):
self._json_parsing_formula = json_parsing_formula
def ToString( self ) -> str:
if self._string_processor.MakesChanges():
full_munge_text = ', applying {}'.format( self._string_processor.ToString() )
else:
full_munge_text = ''
return 'from JSON sidecar{}'.format( full_munge_text )
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_JSON ] = SingleFileMetadataImporterJSON
class SingleFileMetadataImporterTXT( HydrusSerialisable.SerialisableBase, SingleFileMetadataImporterSidecar ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_TXT
SERIALISABLE_NAME = 'Metadata Single File Importer TXT'
SERIALISABLE_VERSION = 2
def __init__( self, string_processor = None, suffix = None ):
if suffix is None:
suffix = ''
if string_processor is None:
string_processor = ClientStrings.StringProcessor()
HydrusSerialisable.SerialisableBase.__init__( self )
SingleFileMetadataImporterSidecar.__init__( self, string_processor, suffix )
def _GetSerialisableInfo( self ):
serialisable_string_processor = self._string_processor.GetSerialisableTuple()
return ( serialisable_string_processor, self._suffix )
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
( serialisable_string_processor, self._suffix ) = serialisable_info
self._string_processor = HydrusSerialisable.CreateFromSerialisableTuple( serialisable_string_processor )
def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
if version == 1:
suffix = old_serialisable_info
string_processor = ClientStrings.StringProcessor()
serialisable_string_processor = string_processor.GetSerialisableTuple()
new_serialisable_info = ( serialisable_string_processor, suffix )
return ( 2, new_serialisable_info )
def GetExpectedSidecarPath( self, actual_file_path: str ):
return ClientMetadataMigrationCore.GetSidecarPath( actual_file_path, self._suffix, 'txt' )
def Import( self, actual_file_path: str ) -> typing.Collection[ str ]:
path = self.GetExpectedSidecarPath( actual_file_path )
if not os.path.exists( path ):
return []
try:
with open( path, 'r', encoding = 'utf-8' ) as f:
raw_text = f.read()
except Exception as e:
raise Exception( 'Could not import from {}: {}'.format( path, str( e ) ) )
rows = HydrusText.DeserialiseNewlinedTexts( raw_text )
if self._string_processor.MakesChanges():
rows = self._string_processor.ProcessStrings( rows )
return rows
def ToString( self ) -> str:
if self._string_processor.MakesChanges():
full_munge_text = ', applying {}'.format( self._string_processor.ToString() )
else:
full_munge_text = ''
return 'from .txt sidecar'.format( full_munge_text )
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_TXT ] = SingleFileMetadataImporterTXT

2
hydrus/core/HydrusConstants.py
View File

@ -80,7 +80,7 @@ options = {}
# Misc
NETWORK_VERSION = 20
SOFTWARE_VERSION = 503
SOFTWARE_VERSION = 504
CLIENT_API_VERSION = 34
SERVER_THUMBNAIL_DIMENSIONS = ( 200, 200 )

1
hydrus/core/HydrusLogger.py
View File

@ -4,7 +4,6 @@ import threading
import time
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
class HydrusLogger( object ):

10
hydrus/core/HydrusSerialisable.py
View File

@ -116,9 +116,15 @@ SERIALISABLE_TYPE_GUI_SESSION_CONTAINER_PAGE_NOTEBOOK = 106
SERIALISABLE_TYPE_GUI_SESSION_CONTAINER_PAGE_SINGLE = 107
SERIALISABLE_TYPE_PRESENTATION_IMPORT_OPTIONS = 108
SERIALISABLE_TYPE_METADATA_SINGLE_FILE_ROUTER = 109
SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_EXPORTER_TXT = 110
SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_EXPORTER_MEDIA_TAGS = 111
SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_TXT = 110
SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_MEDIA_TAGS = 111
SERIALISABLE_TYPE_STRING_TAG_FILTER = 112
SERIALISABLE_TYPE_METADATA_SINGLE_FILE_EXPORTER_JSON = 113
SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_JSON = 114
SERIALISABLE_TYPE_METADATA_SINGLE_FILE_EXPORTER_MEDIA_TAGS = 115
SERIALISABLE_TYPE_METADATA_SINGLE_FILE_EXPORTER_TXT = 116
SERIALISABLE_TYPE_METADATA_SINGLE_FILE_EXPORTER_MEDIA_URLS = 117
SERIALISABLE_TYPE_METADATA_SINGLE_FILE_IMPORTER_MEDIA_URLS = 118
SERIALISABLE_TYPES_TO_OBJECT_TYPES = {}

1
hydrus/core/networking/HydrusNetwork.py
View File

@ -1,5 +1,4 @@
import collections
import itertools
import threading
import time
import typing

6
hydrus/core/networking/HydrusNetworkLegacy.py
View File

@ -1,11 +1,5 @@
import typing
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusSerialisable
from hydrus.core.networking import HydrusNetwork
from hydrus.core.networking import HydrusNetworking
def ConvertToNewAccountType( account_type_key, title, dictionary_string ) -> HydrusNetwork.AccountType:

2
hydrus/core/networking/HydrusNetworkVariableHandling.py
View File

@ -2,7 +2,6 @@ import collections
import json
import os
import traceback
import typing
import urllib
CBOR_AVAILABLE = False
@ -14,7 +13,6 @@ except:
pass
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusFileHandling
from hydrus.core import HydrusImageHandling

4
hydrus/core/networking/HydrusNetworking.py
View File

@ -1,12 +1,9 @@
import calendar
import collections
import datetime
import http.client
import json
import psutil
import socket
import threading
import urllib
import urllib3
from urllib3.exceptions import InsecureRequestWarning
@ -15,7 +12,6 @@ urllib3.disable_warnings( InsecureRequestWarning ) # stopping log-moaning when r
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusSerialisable
# The calendar portion of this works in GMT. A new 'day' or 'month' is calculated based on GMT time, so it won't tick over at midnight for most people.

16
hydrus/test/HelperFunctions.py Normal file
View File

@ -0,0 +1,16 @@
import unittest
def compare_content_updates( ut: unittest.TestCase, service_keys_to_content_updates, expected_service_keys_to_content_updates ):
ut.assertEqual( len( service_keys_to_content_updates ), len( expected_service_keys_to_content_updates ) )
for ( service_key, content_updates ) in service_keys_to_content_updates.items():
expected_content_updates = expected_service_keys_to_content_updates[ service_key ]
c_u_tuples = sorted( ( ( c_u.ToTuple(), c_u.GetReason() ) for c_u in content_updates ) )
e_c_u_tuples = sorted( ( ( e_c_u.ToTuple(), e_c_u.GetReason() ) for e_c_u in expected_content_updates ) )
ut.assertEqual( c_u_tuples, e_c_u_tuples )

2
hydrus/test/TestClientDB.py
View File

@ -1039,7 +1039,7 @@ class TestClientDB( unittest.TestCase ):
service_keys_to_tags = ClientTags.ServiceKeysToTags( { HydrusData.GenerateKey() : [ 'some', 'tags' ] } )
management_controller = ClientGUIManagement.CreateManagementControllerImportHDD( [ 'some', 'paths' ], FileImportOptions.FileImportOptions(), { 'paths' : service_keys_to_tags }, True )
management_controller = ClientGUIManagement.CreateManagementControllerImportHDD( [ 'some', 'paths' ], FileImportOptions.FileImportOptions(), [], { 'paths' : service_keys_to_tags }, True )
management_controller.GetVariable( 'hdd_import' ).PausePlay() # to stop trying to import 'some' 'paths'

680
hydrus/test/TestClientMetadataMigration.py Normal file
View File

@ -0,0 +1,680 @@
import json
import os
import random
import unittest
from hydrus.core import HydrusConstants as HC
from hydrus.core import HydrusData
from hydrus.core import HydrusExceptions
from hydrus.core import HydrusGlobals as HG
from hydrus.core import HydrusText
from hydrus.client import ClientConstants as CC
from hydrus.client import ClientParsing
from hydrus.client import ClientStrings
from hydrus.client.media import ClientMediaManagers
from hydrus.client.media import ClientMediaResult
from hydrus.client.metadata import ClientMetadataMigration
from hydrus.client.metadata import ClientMetadataMigrationExporters
from hydrus.client.metadata import ClientMetadataMigrationImporters
from hydrus.test import HelperFunctions as HF
class TestSingleFileMetadataRouter( unittest.TestCase ):
def test_router( self ):
my_current_storage_tags = { 'samus aran', 'blonde hair' }
my_current_display_tags = { 'character:samus aran', 'blonde hair' }
repo_current_storage_tags = { 'lara croft' }
repo_current_display_tags = { 'character:lara croft' }
repo_pending_storage_tags = { 'tomb raider' }
repo_pending_display_tags = { 'series:tomb raider' }
service_keys_to_statuses_to_storage_tags = {
CC.DEFAULT_LOCAL_TAG_SERVICE_KEY : {
HC.CONTENT_STATUS_CURRENT : my_current_storage_tags
},
HG.test_controller.example_tag_repo_service_key : {
HC.CONTENT_STATUS_CURRENT : repo_current_storage_tags,
HC.CONTENT_STATUS_PENDING : repo_pending_storage_tags
}
}
service_keys_to_statuses_to_display_tags = {
CC.DEFAULT_LOCAL_TAG_SERVICE_KEY : {
HC.CONTENT_STATUS_CURRENT : my_current_display_tags
},
HG.test_controller.example_tag_repo_service_key : {
HC.CONTENT_STATUS_CURRENT : repo_current_display_tags,
HC.CONTENT_STATUS_PENDING : repo_pending_display_tags
}
}
# duplicate to generate proper dicts
tags_manager = ClientMediaManagers.TagsManager(
service_keys_to_statuses_to_storage_tags,
service_keys_to_statuses_to_display_tags
).Duplicate()
#
hash = HydrusData.GenerateKey()
size = 40960
mime = HC.IMAGE_JPEG
width = 640
height = 480
duration = None
num_frames = None
has_audio = False
num_words = None
inbox = True
local_locations_manager = ClientMediaManagers.LocationsManager( { CC.LOCAL_FILE_SERVICE_KEY : 123, CC.COMBINED_LOCAL_FILE_SERVICE_KEY : 123 }, dict(), set(), set(), inbox )
ratings_manager = ClientMediaManagers.RatingsManager( {} )
notes_manager = ClientMediaManagers.NotesManager( {} )
file_viewing_stats_manager = ClientMediaManagers.FileViewingStatsManager.STATICGenerateEmptyManager()
#
file_info_manager = ClientMediaManagers.FileInfoManager( 1, hash, size, mime, width, height, duration, num_frames, has_audio, num_words )
media_result = ClientMediaResult.MediaResult( file_info_manager, tags_manager, local_locations_manager, ratings_manager, notes_manager, file_viewing_stats_manager )
#
actual_file_path = os.path.join( HG.test_controller.db_dir, 'file.jpg' )
expected_output_path = actual_file_path + '.txt'
# empty, works ok but does nothing
router = ClientMetadataMigration.SingleFileMetadataRouter( importers = [], string_processor = None, exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT() )
router.Work( media_result, actual_file_path )
self.assertFalse( os.path.exists( expected_output_path ) )
# doing everything
rows_1 = [ 'character:samus aran', 'blonde hair' ]
rows_2 = [ 'character:lara croft', 'brown hair' ]
expected_input_path_1 = actual_file_path + '.1.txt'
with open( expected_input_path_1, 'w', encoding = 'utf-8' ) as f:
f.write( os.linesep.join( rows_1 ) )
importer_1 = ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT( suffix = '1' )
expected_input_path_2 = actual_file_path + '.2.txt'
with open( expected_input_path_2, 'w', encoding = 'utf-8' ) as f:
f.write( os.linesep.join( rows_2 ) )
importer_2 = ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT( suffix = '2' )
string_processor = ClientStrings.StringProcessor()
processing_steps = [ ClientStrings.StringConverter( conversions = [ ( ClientStrings.STRING_CONVERSION_REMOVE_TEXT_FROM_BEGINNING, 1 ) ] ) ]
string_processor.SetProcessingSteps( processing_steps )
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT()
router = ClientMetadataMigration.SingleFileMetadataRouter( importers = [ importer_1, importer_2 ], string_processor = string_processor, exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT() )
router.Work( media_result, actual_file_path )
self.assertTrue( os.path.exists( expected_output_path ) )
with open( expected_output_path, 'r', encoding = 'utf-8' ) as f:
text = f.read()
os.unlink( expected_output_path )
os.unlink( expected_input_path_1 )
os.unlink( expected_input_path_2 )
result = HydrusText.DeserialiseNewlinedTexts( text )
expected_result = string_processor.ProcessStrings( set( rows_1 ).union( rows_2 ) )
self.assertTrue( len( result ) > 0 )
self.assertEqual( set( result ), set( expected_result ) )
class TestSingleFileMetadataImporters( unittest.TestCase ):
def test_media_tags( self ):
my_current_storage_tags = { 'samus aran', 'blonde hair' }
my_current_display_tags = { 'character:samus aran', 'blonde hair' }
repo_current_storage_tags = { 'lara croft' }
repo_current_display_tags = { 'character:lara croft' }
repo_pending_storage_tags = { 'tomb raider' }
repo_pending_display_tags = { 'series:tomb raider' }
service_keys_to_statuses_to_storage_tags = {
CC.DEFAULT_LOCAL_TAG_SERVICE_KEY : {
HC.CONTENT_STATUS_CURRENT : my_current_storage_tags
},
HG.test_controller.example_tag_repo_service_key : {
HC.CONTENT_STATUS_CURRENT : repo_current_storage_tags,
HC.CONTENT_STATUS_PENDING : repo_pending_storage_tags
}
}
service_keys_to_statuses_to_display_tags = {
CC.DEFAULT_LOCAL_TAG_SERVICE_KEY : {
HC.CONTENT_STATUS_CURRENT : my_current_display_tags
},
HG.test_controller.example_tag_repo_service_key : {
HC.CONTENT_STATUS_CURRENT : repo_current_display_tags,
HC.CONTENT_STATUS_PENDING : repo_pending_display_tags
}
}
# duplicate to generate proper dicts
tags_manager = ClientMediaManagers.TagsManager(
service_keys_to_statuses_to_storage_tags,
service_keys_to_statuses_to_display_tags
).Duplicate()
#
hash = HydrusData.GenerateKey()
size = 40960
mime = HC.IMAGE_JPEG
width = 640
height = 480
duration = None
num_frames = None
has_audio = False
num_words = None
inbox = True
local_locations_manager = ClientMediaManagers.LocationsManager( { CC.LOCAL_FILE_SERVICE_KEY : 123, CC.COMBINED_LOCAL_FILE_SERVICE_KEY : 123 }, dict(), set(), set(), inbox )
ratings_manager = ClientMediaManagers.RatingsManager( {} )
notes_manager = ClientMediaManagers.NotesManager( {} )
file_viewing_stats_manager = ClientMediaManagers.FileViewingStatsManager.STATICGenerateEmptyManager()
#
file_info_manager = ClientMediaManagers.FileInfoManager( 1, hash, size, mime, width, height, duration, num_frames, has_audio, num_words )
media_result = ClientMediaResult.MediaResult( file_info_manager, tags_manager, local_locations_manager, ratings_manager, notes_manager, file_viewing_stats_manager )
# simple local
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaTags( service_key = CC.DEFAULT_LOCAL_TAG_SERVICE_KEY )
result = importer.Import( media_result )
self.assertEqual( set( result ), set( my_current_storage_tags ) )
# simple repo
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaTags( service_key = HG.test_controller.example_tag_repo_service_key )
result = importer.Import( media_result )
self.assertEqual( set( result ), set( repo_current_storage_tags ) )
# all known
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaTags( service_key = CC.COMBINED_TAG_SERVICE_KEY )
result = importer.Import( media_result )
self.assertEqual( set( result ), set( my_current_storage_tags ).union( repo_current_storage_tags ) )
# with string processor
string_processor = ClientStrings.StringProcessor()
processing_steps = [ ClientStrings.StringConverter( conversions = [ ( ClientStrings.STRING_CONVERSION_REMOVE_TEXT_FROM_BEGINNING, 1 ) ] ) ]
string_processor.SetProcessingSteps( processing_steps )
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaTags( string_processor = string_processor, service_key = CC.DEFAULT_LOCAL_TAG_SERVICE_KEY )
result = importer.Import( media_result )
self.assertTrue( len( result ) > 0 )
self.assertNotEqual( set( result ), set( my_current_storage_tags ) )
self.assertEqual( set( result ), set( string_processor.ProcessStrings( my_current_storage_tags ) ) )
def test_media_urls( self ):
urls = { 'https://site.com/123456', 'https://cdn5.st.com/file/123456' }
# simple
hash = HydrusData.GenerateKey()
size = 40960
mime = HC.IMAGE_JPEG
width = 640
height = 480
duration = None
num_frames = None
has_audio = False
num_words = None
inbox = True
local_locations_manager = ClientMediaManagers.LocationsManager( { CC.LOCAL_FILE_SERVICE_KEY : 123, CC.COMBINED_LOCAL_FILE_SERVICE_KEY : 123 }, dict(), set(), set(), inbox, urls )
# duplicate to generate proper dicts
tags_manager = ClientMediaManagers.TagsManager( {}, {} ).Duplicate()
ratings_manager = ClientMediaManagers.RatingsManager( {} )
notes_manager = ClientMediaManagers.NotesManager( {} )
file_viewing_stats_manager = ClientMediaManagers.FileViewingStatsManager.STATICGenerateEmptyManager()
#
file_info_manager = ClientMediaManagers.FileInfoManager( 1, hash, size, mime, width, height, duration, num_frames, has_audio, num_words )
media_result = ClientMediaResult.MediaResult( file_info_manager, tags_manager, local_locations_manager, ratings_manager, notes_manager, file_viewing_stats_manager )
# simple
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaURLs()
result = importer.Import( media_result )
self.assertEqual( set( result ), set( urls ) )
# with string processor
string_processor = ClientStrings.StringProcessor()
processing_steps = [ ClientStrings.StringConverter( conversions = [ ( ClientStrings.STRING_CONVERSION_REMOVE_TEXT_FROM_BEGINNING, 1 ) ] ) ]
string_processor.SetProcessingSteps( processing_steps )
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterMediaURLs( string_processor = string_processor )
result = importer.Import( media_result )
self.assertTrue( len( result ) > 0 )
self.assertNotEqual( set( result ), set( urls ) )
self.assertEqual( set( result ), set( string_processor.ProcessStrings( urls ) ) )
def test_media_txt( self ):
actual_file_path = os.path.join( HG.test_controller.db_dir, 'file.jpg' )
rows = [ 'character:samus aran', 'blonde hair' ]
# simple
expected_input_path = actual_file_path + '.txt'
with open( expected_input_path, 'w', encoding = 'utf-8' ) as f:
f.write( os.linesep.join( rows ) )
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT()
result = importer.Import( actual_file_path )
os.unlink( expected_input_path )
self.assertEqual( set( result ), set( rows ) )
# with suffix and processing
string_processor = ClientStrings.StringProcessor()
processing_steps = [ ClientStrings.StringConverter( conversions = [ ( ClientStrings.STRING_CONVERSION_REMOVE_TEXT_FROM_BEGINNING, 1 ) ] ) ]
string_processor.SetProcessingSteps( processing_steps )
expected_input_path = actual_file_path + '.tags.txt'
with open( expected_input_path, 'w', encoding = 'utf-8' ) as f:
f.write( os.linesep.join( rows ) )
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterTXT( string_processor = string_processor, suffix = 'tags' )
result = importer.Import( actual_file_path )
os.unlink( expected_input_path )
self.assertTrue( len( result ) > 0 )
self.assertNotEqual( set( result ), set( rows ) )
self.assertEqual( set( result ), set( string_processor.ProcessStrings( rows ) ) )
def test_media_json( self ):
actual_file_path = os.path.join( HG.test_controller.db_dir, 'file.jpg' )
rows = [ 'character:samus aran', 'blonde hair' ]
# no file means no rows
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterJSON()
result = importer.Import( actual_file_path )
self.assertEqual( set( result ), set() )
# simple
expected_input_path = actual_file_path + '.json'
with open( expected_input_path, 'w', encoding = 'utf-8' ) as f:
j = json.dumps( rows )
f.write( j )
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterJSON()
result = importer.Import( actual_file_path )
os.unlink( expected_input_path )
self.assertEqual( set( result ), set( rows ) )
# with suffix, processing, and dest
string_processor = ClientStrings.StringProcessor()
processing_steps = [ ClientStrings.StringConverter( conversions = [ ( ClientStrings.STRING_CONVERSION_REMOVE_TEXT_FROM_BEGINNING, 1 ) ] ) ]
string_processor.SetProcessingSteps( processing_steps )
expected_input_path = actual_file_path + '.tags.json'
with open( expected_input_path, 'w', encoding = 'utf-8' ) as f:
d = { 'file_data' : { 'tags' : rows } }
j = json.dumps( d )
f.write( j )
parse_rules = [
( ClientParsing.JSON_PARSE_RULE_TYPE_DICT_KEY, ClientStrings.StringMatch( match_type = ClientStrings.STRING_MATCH_FIXED, match_value = 'file_data', example_string = 'file_data' ) ),
( ClientParsing.JSON_PARSE_RULE_TYPE_DICT_KEY, ClientStrings.StringMatch( match_type = ClientStrings.STRING_MATCH_FIXED, match_value = 'tags', example_string = 'tags' ) ),
( ClientParsing.JSON_PARSE_RULE_TYPE_ALL_ITEMS, None )
]
json_parsing_formula = ClientParsing.ParseFormulaJSON( parse_rules = parse_rules, content_to_fetch = ClientParsing.JSON_CONTENT_STRING )
importer = ClientMetadataMigrationImporters.SingleFileMetadataImporterJSON( string_processor = string_processor, suffix = 'tags', json_parsing_formula = json_parsing_formula )
result = importer.Import( actual_file_path )
os.unlink( expected_input_path )
self.assertTrue( len( result ) > 0 )
self.assertNotEqual( set( result ), set( rows ) )
self.assertEqual( set( result ), set( string_processor.ProcessStrings( rows ) ) )
class TestSingleFileMetadataExporters( unittest.TestCase ):
def test_media_tags( self ):
hash = os.urandom( 32 )
rows = [ 'character:samus aran', 'blonde hair' ]
# no tags makes no write
service_key = HG.test_controller.example_tag_repo_service_key
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags( service_key )
HG.test_controller.ClearWrites( 'content_updates' )
exporter.Export( hash, [] )
with self.assertRaises( Exception ):
[ ( ( service_keys_to_content_updates, ), kwargs ) ] = HG.test_controller.GetWrite( 'content_updates' )
# simple local
service_key = CC.DEFAULT_LOCAL_TAG_SERVICE_KEY
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags( service_key )
HG.test_controller.ClearWrites( 'content_updates' )
exporter.Export( hash, rows )
hashes = { hash }
expected_service_keys_to_content_updates = { service_key : [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ( tag, hashes ) ) for tag in rows ] }
[ ( ( service_keys_to_content_updates, ), kwargs ) ] = HG.test_controller.GetWrite( 'content_updates' )
HF.compare_content_updates( self, service_keys_to_content_updates, expected_service_keys_to_content_updates )
# simple repo
service_key = HG.test_controller.example_tag_repo_service_key
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaTags( service_key )
HG.test_controller.ClearWrites( 'content_updates' )
exporter.Export( hash, rows )
hashes = { hash }
expected_service_keys_to_content_updates = { service_key : [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ( tag, hashes ) ) for tag in rows ] }
[ ( ( service_keys_to_content_updates, ), kwargs ) ] = HG.test_controller.GetWrite( 'content_updates' )
HF.compare_content_updates( self, service_keys_to_content_updates, expected_service_keys_to_content_updates )
def test_media_urls( self ):
hash = os.urandom( 32 )
urls = [ 'https://site.com/123456', 'https://cdn5.st.com/file/123456' ]
# no urls makes no write
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaURLs()
HG.test_controller.ClearWrites( 'content_updates' )
exporter.Export( hash, [] )
with self.assertRaises( Exception ):
[ ( ( service_keys_to_content_updates, ), kwargs ) ] = HG.test_controller.GetWrite( 'content_updates' )
# simple
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterMediaURLs()
HG.test_controller.ClearWrites( 'content_updates' )
exporter.Export( hash, urls )
expected_service_keys_to_content_updates = { CC.COMBINED_LOCAL_FILE_SERVICE_KEY : [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( urls, { hash } ) ) ] }
[ ( ( service_keys_to_content_updates, ), kwargs ) ] = HG.test_controller.GetWrite( 'content_updates' )
HF.compare_content_updates( self, service_keys_to_content_updates, expected_service_keys_to_content_updates )
def test_media_txt( self ):
actual_file_path = os.path.join( HG.test_controller.db_dir, 'file.jpg' )
rows = [ 'character:samus aran', 'blonde hair' ]
# no rows makes no write
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT()
exporter.Export( actual_file_path, [] )
expected_output_path = actual_file_path + '.txt'
self.assertFalse( os.path.exists( expected_output_path ) )
# simple
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT()
exporter.Export( actual_file_path, rows )
expected_output_path = actual_file_path + '.txt'
self.assertTrue( os.path.exists( expected_output_path ) )
with open( expected_output_path, 'r', encoding = 'utf-8' ) as f:
text = f.read()
os.unlink( expected_output_path )
self.assertEqual( set( rows ), set( HydrusText.DeserialiseNewlinedTexts( text ) ) )
# with suffix
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterTXT( suffix = 'tags' )
exporter.Export( actual_file_path, rows )
expected_output_path = actual_file_path + '.tags.txt'
self.assertTrue( os.path.exists( expected_output_path ) )
with open( expected_output_path, 'r', encoding = 'utf-8' ) as f:
text = f.read()
os.unlink( expected_output_path )
self.assertEqual( set( rows ), set( HydrusText.DeserialiseNewlinedTexts( text ) ) )
def test_media_json( self ):
actual_file_path = os.path.join( HG.test_controller.db_dir, 'file.jpg' )
rows = [ 'character:samus aran', 'blonde hair' ]
# no rows makes no write
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterJSON()
exporter.Export( actual_file_path, [] )
expected_output_path = actual_file_path + '.json'
self.assertFalse( os.path.exists( expected_output_path ) )
# simple
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterJSON()
exporter.Export( actual_file_path, rows )
expected_output_path = actual_file_path + '.json'
self.assertTrue( os.path.exists( expected_output_path ) )
with open( expected_output_path, 'r', encoding = 'utf-8' ) as f:
text = f.read()
os.unlink( expected_output_path )
self.assertEqual( set( rows ), set( json.loads( text ) ) )
# with suffix and json dest
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterJSON( suffix = 'tags', nested_object_names = [ 'file_data', 'tags' ] )
exporter.Export( actual_file_path, rows )
expected_output_path = actual_file_path + '.tags.json'
self.assertTrue( os.path.exists( expected_output_path ) )
with open( expected_output_path, 'r', encoding = 'utf-8' ) as f:
text = f.read()
os.unlink( expected_output_path )
self.assertEqual( set( rows ), set( json.loads( text )[ 'file_data' ][ 'tags' ] ) )
def test_media_json_combined( self ):
actual_file_path = os.path.join( HG.test_controller.db_dir, 'file.jpg' )
#
tag_rows = [ 'character:samus aran', 'blonde hair' ]
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterJSON( nested_object_names = [ 'file_data', 'tags' ] )
exporter.Export( actual_file_path, tag_rows )
#
url_rows = [ 'https://site.com/123456' ]
exporter = ClientMetadataMigrationExporters.SingleFileMetadataExporterJSON( nested_object_names = [ 'file_data', 'urls' ] )
exporter.Export( actual_file_path, url_rows )
#
expected_output_path = actual_file_path + '.json'
self.assertTrue( os.path.exists( expected_output_path ) )
with open( expected_output_path, 'r', encoding = 'utf-8' ) as f:
text = f.read()
os.unlink( expected_output_path )
self.assertEqual( set( tag_rows ), set( json.loads( text )[ 'file_data' ][ 'tags' ] ) )
self.assertEqual( set( url_rows ), set( json.loads( text )[ 'file_data' ][ 'urls' ] ) )

8
hydrus/test/TestController.py
View File

@ -52,6 +52,7 @@ from hydrus.test import TestClientImageHandling
from hydrus.test import TestClientImportOptions
from hydrus.test import TestClientImportSubscriptions
from hydrus.test import TestClientListBoxes
from hydrus.test import TestClientMetadataMigration
from hydrus.test import TestClientMigration
from hydrus.test import TestClientNetworking
from hydrus.test import TestClientParsing
@ -780,6 +781,7 @@ class Controller( object ):
TestHydrusNetworking,
TestClientImportSubscriptions,
TestClientImageHandling,
TestClientMetadataMigration,
TestClientMigration,
TestHydrusServer
]
@ -788,7 +790,7 @@ class Controller( object ):
TestDialogs,
TestClientListBoxes
]
module_lookup[ 'client_api' ] = [
TestClientAPI
]
@ -857,6 +859,10 @@ class Controller( object ):
TestClientImageHandling
]
module_lookup[ 'metadata_migration' ] = [
TestClientMetadataMigration
]
module_lookup[ 'migration' ] = [
TestClientMigration
]

28
requirements_old_mpv.txt Normal file
View File

@ -0,0 +1,28 @@
cbor2
python-dateutil
beautifulsoup4>=4.0.0
chardet>=3.0.4
cloudscraper>=1.2.33
html5lib>=1.0.1
lxml>=4.5.0
lz4>=3.0.0
nose>=1.3.0
numpy>=1.16.0
Pillow>=6.0.0
psutil>=5.0.0
pyOpenSSL>=19.1.0
PySocks>=1.7.0
PyYAML>=5.0.0
Send2Trash>=1.5.0
service-identity>=18.1.0
six>=1.14.0
Twisted>=20.3.0
opencv-python-headless==4.5.3.56
python-mpv==0.5.2
QtPy==2.2.1
requests==2.28.1
setuptools==65.4.1
PySide6==6.3.2

BIN
static/default/gugs/deviant art tag search.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 1.7 KiB

BIN
static/hydrus_128.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 19 KiB