RepoMirrors
/
hydrus
mirror of https://github.com/hydrusnetwork/hydrus.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
hydrus/Fixing_Hydrus_Random_Crashe...

2453 lines
71 KiB
HTML
Raw Blame History

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="description" content="A personal booru-style media tagger that can import files and tags from your hard drive and popular websites. Content can be shared with other users via user-run servers.">
<link rel="canonical" href="https://hydrusnetwork.github.io/hydrus/Fixing_Hydrus_Random_Crashes_Under_Linux.html">
<link rel="icon" href="assets/favicon.svg">
<meta name="generator" content="mkdocs-1.5.3, mkdocs-material-9.5.15">
<title>Virtual Memory Under Linux - hydrus network</title>
<link rel="stylesheet" href="assets/stylesheets/main.7e359304.min.css">
<link rel="stylesheet" href="assets/stylesheets/palette.06af60db.min.css">
<link rel="stylesheet" href="assets/stylesheets/extra.css">
<script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce((e,_)=>(e<<5)-e+_.charCodeAt(0),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="blue-grey" data-md-color-accent="blue">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#virtual_memory_under_linux" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href="index.html" title="hydrus network" class="md-header__button md-logo" aria-label="hydrus network" data-md-component="logo">
<img src="assets/hydrus-white.svg" alt="logo">
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
hydrus network
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Virtual Memory Under Linux
</span>
</div>
</div>
</div>
<form class="md-header__option" data-md-component="palette">
<input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="blue-grey" data-md-color-accent="blue" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_0">
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_1" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 6H7c-3.31 0-6 2.69-6 6s2.69 6 6 6h10c3.31 0 6-2.69 6-6s-2.69-6-6-6zm0 10H7c-2.21 0-4-1.79-4-4s1.79-4 4-4h10c2.21 0 4 1.79 4 4s-1.79 4-4 4zM7 9c-1.66 0-3 1.34-3 3s1.34 3 3 3 3-1.34 3-3-1.34-3-3-3z"/></svg>
</label>
<input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="black" data-md-color-accent="blue" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_1">
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_0" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 7H7a5 5 0 0 0-5 5 5 5 0 0 0 5 5h10a5 5 0 0 0 5-5 5 5 0 0 0-5-5m0 8a3 3 0 0 1-3-3 3 3 0 0 1 3-3 3 3 0 0 1 3 3 3 3 0 0 1-3 3Z"/></svg>
</label>
</form>
<script>var media,input,key,value,palette=__md_get("__palette");if(palette&&palette.color){"(prefers-color-scheme)"===palette.color.media&&(media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']"),palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent"));for([key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
</button>
</nav>
<div class="md-search__suggest" data-md-component="search-suggest"></div>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
<div class="md-header__source">
<a href="https://github.com/hydrusnetwork/hydrus" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
</div>
<div class="md-source__repository">
hydrusnetwork/hydrus
</div>
</a>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item">
<a href="index.html" class="md-tabs__link">
Home
</a>
</li>
<li class="md-tabs__item">
<a href="introduction.html" class="md-tabs__link">
Getting Started
</a>
</li>
<li class="md-tabs__item">
<a href="advanced_siblings.html" class="md-tabs__link">
Advanced
</a>
</li>
<li class="md-tabs__item">
<a href="client_api.html" class="md-tabs__link">
API
</a>
</li>
<li class="md-tabs__item">
<a href="faq.html" class="md-tabs__link">
Misc
</a>
</li>
</ul>
</div>
</nav>
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" hidden>
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="index.html" title="hydrus network" class="md-nav__button md-logo" aria-label="hydrus network" data-md-component="logo">
<img src="assets/hydrus-white.svg" alt="logo">
</a>
hydrus network
</label>
<div class="md-nav__source">
<a href="https://github.com/hydrusnetwork/hydrus" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
</div>
<div class="md-source__repository">
hydrusnetwork/hydrus
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="index.html" class="md-nav__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" >
<label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="0">
<span class="md-ellipsis">
Getting Started
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2">
<span class="md-nav__icon md-icon"></span>
Getting Started
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="introduction.html" class="md-nav__link">
<span class="md-ellipsis">
Introduction and Statement of Principles
</span>
</a>
</li>
<li class="md-nav__item">
<a href="gettingStartedOverview.html" class="md-nav__link">
<span class="md-ellipsis">
Overview For Getting Started
</span>
</a>
</li>
<li class="md-nav__item">
<a href="getting_started_installing.html" class="md-nav__link">
<span class="md-ellipsis">
Installing and Updating
</span>
</a>
</li>
<li class="md-nav__item">
<a href="getting_started_files.html" class="md-nav__link">
<span class="md-ellipsis">
Files
</span>
</a>
</li>
<li class="md-nav__item">
<a href="filetypes.html" class="md-nav__link">
<span class="md-ellipsis">
Supported Filetypes
</span>
</a>
</li>
<li class="md-nav__item">
<a href="getting_started_importing.html" class="md-nav__link">
<span class="md-ellipsis">
Importing and Exporting
</span>
</a>
</li>
<li class="md-nav__item">
<a href="getting_started_tags.html" class="md-nav__link">
<span class="md-ellipsis">
Tags
</span>
</a>
</li>
<li class="md-nav__item">
<a href="getting_started_searching.html" class="md-nav__link">
<span class="md-ellipsis">
Searching and Sorting
</span>
</a>
</li>
<li class="md-nav__item">
<a href="getting_started_more_tags.html" class="md-nav__link">
<span class="md-ellipsis">
More Tags
</span>
</a>
</li>
<li class="md-nav__item">
<a href="getting_started_downloading.html" class="md-nav__link">
<span class="md-ellipsis">
Downloading
</span>
</a>
</li>
<li class="md-nav__item">
<a href="getting_started_ratings.html" class="md-nav__link">
<span class="md-ellipsis">
Ratings
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_12" >
<label class="md-nav__link" for="__nav_2_12" id="__nav_2_12_label" tabindex="0">
<span class="md-ellipsis">
PTR
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_12_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_12">
<span class="md-nav__icon md-icon"></span>
PTR
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="access_keys.html" class="md-nav__link">
<span class="md-ellipsis">
PTR Access Keys
</span>
</a>
</li>
<li class="md-nav__item">
<a href="PTR.html" class="md-nav__link">
<span class="md-ellipsis">
PTR Guide
</span>
</a>
</li>
<li class="md-nav__item">
<a href="petitionPractices.html" class="md-nav__link">
<span class="md-ellipsis">
Petition Practices
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_13" >
<label class="md-nav__link" for="__nav_2_13" id="__nav_2_13_label" tabindex="0">
<span class="md-ellipsis">
Next Steps
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_13_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2_13">
<span class="md-nav__icon md-icon"></span>
Next Steps
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="adding_new_downloaders.html" class="md-nav__link">
<span class="md-ellipsis">
Adding New Downloaders
</span>
</a>
</li>
<li class="md-nav__item">
<a href="getting_started_subscriptions.html" class="md-nav__link">
<span class="md-ellipsis">
Subscriptions
</span>
</a>
</li>
<li class="md-nav__item">
<a href="duplicates.html" class="md-nav__link">
<span class="md-ellipsis">
Filtering Duplicates
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
<span class="md-ellipsis">
Advanced
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3">
<span class="md-nav__icon md-icon"></span>
Advanced
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="advanced_siblings.html" class="md-nav__link">
<span class="md-ellipsis">
Tag Siblings
</span>
</a>
</li>
<li class="md-nav__item">
<a href="advanced_parents.html" class="md-nav__link">
<span class="md-ellipsis">
Tag Parents
</span>
</a>
</li>
<li class="md-nav__item">
<a href="advanced_sidecars.html" class="md-nav__link">
<span class="md-ellipsis">
Sidecars
</span>
</a>
</li>
<li class="md-nav__item">
<a href="advanced_multiple_local_file_services.html" class="md-nav__link">
<span class="md-ellipsis">
Multiple Local File Services
</span>
</a>
</li>
<li class="md-nav__item">
<a href="advanced.html" class="md-nav__link">
<span class="md-ellipsis">
General Clever Tricks
</span>
</a>
</li>
<li class="md-nav__item">
<a href="reducing_lag.html" class="md-nav__link">
<span class="md-ellipsis">
Reducing Lag
</span>
</a>
</li>
<li class="md-nav__item">
<a href="database_migration.html" class="md-nav__link">
<span class="md-ellipsis">
Database Migration
</span>
</a>
</li>
<li class="md-nav__item">
<a href="launch_arguments.html" class="md-nav__link">
<span class="md-ellipsis">
Launch Arguments
</span>
</a>
</li>
<li class="md-nav__item">
<a href="ipfs.html" class="md-nav__link">
<span class="md-ellipsis">
IPFS
</span>
</a>
</li>
<li class="md-nav__item">
<a href="local_booru.html" class="md-nav__link">
<span class="md-ellipsis">
Local Booru
</span>
</a>
</li>
<li class="md-nav__item">
<a href="server.html" class="md-nav__link">
<span class="md-ellipsis">
Running Your Own Server
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_12" >
<label class="md-nav__link" for="__nav_3_12" id="__nav_3_12_label" tabindex="0">
<span class="md-ellipsis">
Alternate Installations
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_12_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_12">
<span class="md-nav__icon md-icon"></span>
Alternate Installations
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="docker.html" class="md-nav__link">
<span class="md-ellipsis">
Docker
</span>
</a>
</li>
<li class="md-nav__item">
<a href="wine.html" class="md-nav__link">
<span class="md-ellipsis">
Running In Wine
</span>
</a>
</li>
<li class="md-nav__item">
<a href="running_from_source.html" class="md-nav__link">
<span class="md-ellipsis">
Running From Source
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_13" >
<label class="md-nav__link" for="__nav_3_13" id="__nav_3_13_label" tabindex="0">
<span class="md-ellipsis">
Downloader Creation
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_13_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_13">
<span class="md-nav__icon md-icon"></span>
Downloader Creation
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="downloader_intro.html" class="md-nav__link">
<span class="md-ellipsis">
Introduction
</span>
</a>
</li>
<li class="md-nav__item">
<a href="downloader_gugs.html" class="md-nav__link">
<span class="md-ellipsis">
Gallery URL Generators
</span>
</a>
</li>
<li class="md-nav__item">
<a href="downloader_url_classes.html" class="md-nav__link">
<span class="md-ellipsis">
URL Classes
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_13_4" >
<label class="md-nav__link" for="__nav_3_13_4" id="__nav_3_13_4_label" tabindex="0">
<span class="md-ellipsis">
Parsers
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_13_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_13_4">
<span class="md-nav__icon md-icon"></span>
Parsers
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="downloader_parsers.html" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_13_4_2" >
<label class="md-nav__link" for="__nav_3_13_4_2" id="__nav_3_13_4_2_label" tabindex="0">
<span class="md-ellipsis">
Components
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="4" aria-labelledby="__nav_3_13_4_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_13_4_2">
<span class="md-nav__icon md-icon"></span>
Components
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="downloader_parsers_formulae.html" class="md-nav__link">
<span class="md-ellipsis">
Formulae
</span>
</a>
</li>
<li class="md-nav__item">
<a href="downloader_parsers_content_parsers.html" class="md-nav__link">
<span class="md-ellipsis">
Content Parsers
</span>
</a>
</li>
<li class="md-nav__item">
<a href="downloader_parsers_page_parsers.html" class="md-nav__link">
<span class="md-ellipsis">
Page Parsers
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_13_4_3" >
<label class="md-nav__link" for="__nav_3_13_4_3" id="__nav_3_13_4_3_label" tabindex="0">
<span class="md-ellipsis">
Walkthroughs
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="4" aria-labelledby="__nav_3_13_4_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3_13_4_3">
<span class="md-nav__icon md-icon"></span>
Walkthroughs
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="downloader_parsers_full_example_gallery_page.html" class="md-nav__link">
<span class="md-ellipsis">
Gallery Page Example
</span>
</a>
</li>
<li class="md-nav__item">
<a href="downloader_parsers_full_example_file_page.html" class="md-nav__link">
<span class="md-ellipsis">
File Page Example
</span>
</a>
</li>
<li class="md-nav__item">
<a href="downloader_parsers_full_example_api.html" class="md-nav__link">
<span class="md-ellipsis">
API Example
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="downloader_completion.html" class="md-nav__link">
<span class="md-ellipsis">
Putting It All Together
</span>
</a>
</li>
<li class="md-nav__item">
<a href="downloader_sharing.html" class="md-nav__link">
<span class="md-ellipsis">
Sharing
</span>
</a>
</li>
<li class="md-nav__item">
<a href="downloader_login.html" class="md-nav__link">
<span class="md-ellipsis">
Login Manager
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" >
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
<span class="md-ellipsis">
API
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4">
<span class="md-nav__icon md-icon"></span>
API
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="client_api.html" class="md-nav__link">
<span class="md-ellipsis">
Client API
</span>
</a>
</li>
<li class="md-nav__item">
<a href="developer_api.html" class="md-nav__link">
<span class="md-ellipsis">
API documentation
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
<span class="md-ellipsis">
Misc
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_5">
<span class="md-nav__icon md-icon"></span>
Misc
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="faq.html" class="md-nav__link">
<span class="md-ellipsis">
FAQ
</span>
</a>
</li>
<li class="md-nav__item">
<a href="privacy.html" class="md-nav__link">
<span class="md-ellipsis">
Privacy
</span>
</a>
</li>
<li class="md-nav__item">
<a href="contact.html" class="md-nav__link">
<span class="md-ellipsis">
Contact and Links
</span>
</a>
</li>
<li class="md-nav__item">
<a href="support.html" class="md-nav__link">
<span class="md-ellipsis">
Financial Support
</span>
</a>
</li>
<li class="md-nav__item">
<a href="changelog.html" class="md-nav__link">
<span class="md-ellipsis">
Changelog
</span>
</a>
</li>
<li class="md-nav__item">
<a href="about_docs.html" class="md-nav__link">
<span class="md-ellipsis">
About These Docs
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#why_does_hydrus_keep_crashing_under_linux_when_it_has_lots_of_virtual_memory" class="md-nav__link">
<span class="md-ellipsis">
Why does hydrus keep crashing under Linux when it has lots of virtual memory?
</span>
</a>
<nav class="md-nav" aria-label="Why does hydrus keep crashing under Linux when it has lots of virtual memory?">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#symptoms" class="md-nav__link">
<span class="md-ellipsis">
Symptoms
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#tldr_the_fix" class="md-nav__link">
<span class="md-ellipsis">
tl;dr :: The fix
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#details" class="md-nav__link">
<span class="md-ellipsis">
Details
</span>
</a>
<nav class="md-nav" aria-label="Details">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#the_oom_killer" class="md-nav__link">
<span class="md-ellipsis">
The OOM Killer
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#memory_overcommmit" class="md-nav__link">
<span class="md-ellipsis">
Memory Overcommmit
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#what_about_swappiness" class="md-nav__link">
<span class="md-ellipsis">
What about swappiness?
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#why_does_my_linux_system_studder_or_become_unresponsive_when_hydrus_has_been_running_a_while" class="md-nav__link">
<span class="md-ellipsis">
Why does my Linux system studder or become unresponsive when hydrus has been running a while?
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#why_does_everything_become_clunky_for_a_bit_if_i_have_tuned_all_of_the_above_settings_especially_if_i_try_to_do_something_on_the_system_that_isnt_hydrus" class="md-nav__link">
<span class="md-ellipsis">
Why does everything become clunky for a bit if I have tuned all of the above settings? (especially if I try to do something on the system that isn't hydrus)
</span>
</a>
<nav class="md-nav" aria-label="Why does everything become clunky for a bit if I have tuned all of the above settings? (especially if I try to do something on the system that isn't hydrus)">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#virtual_memory_under_linux_4_unleash_the_memory" class="md-nav__link">
<span class="md-ellipsis">
Virtual Memory Under Linux 4: Unleash the memory
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#virtual_memory_under_linux_5_phenomenal_cosmic_power_itty_bitty_living_space" class="md-nav__link">
<span class="md-ellipsis">
Virtual Memory Under Linux 5: Phenomenal Cosmic Power; Itty bitty living space
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<a href="https://github.com/hydrusnetwork/hydrus/edit/master/docs/Fixing_Hydrus_Random_Crashes_Under_Linux.md" title="Edit this page" class="md-content__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M10 20H6V4h7v5h5v3.1l2-2V8l-6-6H6c-1.1 0-2 .9-2 2v16c0 1.1.9 2 2 2h4v-2m10.2-7c.1 0 .3.1.4.2l1.3 1.3c.2.2.2.6 0 .8l-1 1-2.1-2.1 1-1c.1-.1.2-.2.4-.2m0 3.9L14.1 23H12v-2.1l6.1-6.1 2.1 2.1Z"/></svg>
</a>
<h1 id="virtual_memory_under_linux">Virtual Memory Under Linux<a class="headerlink" href="#virtual_memory_under_linux" title="Permanent link">&para;</a></h1>
<h2 id="why_does_hydrus_keep_crashing_under_linux_when_it_has_lots_of_virtual_memory">Why does hydrus keep crashing under Linux when it has lots of virtual memory?<a class="headerlink" href="#why_does_hydrus_keep_crashing_under_linux_when_it_has_lots_of_virtual_memory" title="Permanent link">&para;</a></h2>
<h3 id="symptoms">Symptoms<a class="headerlink" href="#symptoms" title="Permanent link">&para;</a></h3>
<ul>
<li>Hydrus crashes without a crash log</li>
<li>Standard error reads <code>Killed</code></li>
<li>System logs say OOMKiller</li>
<li>Programs appear to havevery high virtual memory utilization despite low real memory.</li>
</ul>
<h3 id="tldr_the_fix">tl;dr :: The fix<a class="headerlink" href="#tldr_the_fix" title="Permanent link">&para;</a></h3>
<p>Add the followng line to the end of <code>/etc/sysctl.conf</code>. You will need admin, so use</p>
<p><code>sudo nano /etc/sysctl.conf</code> or
<code>sudo gedit /etc/sysctl.conf</code></p>
<div class="highlight"><pre><span></span><code><span class="na">vm.min_free_kbytes</span><span class="o">=</span><span class="s">1153434</span>
<span class="na">vm.overcommit_memory</span><span class="o">=</span><span class="s">1</span>
</code></pre></div>
<p>Check that you have (enough) swap space or you might still run out of memory.
<div class="highlight"><pre><span></span><code>sudo<span class="w"> </span>swapon<span class="w"> </span>--show
</code></pre></div></p>
<p>If you need swap
<div class="highlight"><pre><span></span><code>sudo<span class="w"> </span>fallocate<span class="w"> </span>-l<span class="w"> </span>16G<span class="w"> </span>/swapfile<span class="w"> </span><span class="c1">#make 16GiB of swap</span>
sudo<span class="w"> </span>chmod<span class="w"> </span><span class="m">600</span><span class="w"> </span>/swapfile
sudo<span class="w"> </span>mkswap<span class="w"> </span>/swapfile
</code></pre></div>
Add to <code>/etc/fstab</code> so your swap is mounted on reboot
<div class="highlight"><pre><span></span><code>/swapfile swap swap defaults 0 0
</code></pre></div></p>
<p>You may add as many swapfiles as you like, and should add a new swapfile before you delete an old one if you plan to do so, as unmounting a swapfile will evict its contents back in to real memory. You may also wish to use a swapfile type that uses compression, this saves you some disk space for a little bit of a performance hit, but also significantly saves on mostly empty memory.</p>
<p>Reboot for all changes to take effect, or use <code>sysctl</code> to set <code>vm</code> variables.</p>
<h3 id="details">Details<a class="headerlink" href="#details" title="Permanent link">&para;</a></h3>
<p>Linux's memory allocator is lazy and does not perform opportunistic reclaim. This means that the system will continue to give your process memory from the real and virtual memory pool(swap) until there is none left.</p>
<p>Linux will only cleanup if the available total real and virtual memory falls below the <strong>watermark</strong> as defined in the system control configuration file <code>/etc/sysctl.conf</code>.
The watermark's name is <code>vm.min_free_kbytes</code>, it is the number of kilobytes the system keeps in reserve, and therefore the maximum amount of memory the system can allocate in one go before needing to reclaim memory it gave eariler but which is no longer in use.</p>
<p>The default value is <code>vm.min_free_kbytes=65536</code>, which means 66MiB (megabytes). </p>
<p>If for a given request the amount of memory asked to be allocated is under <code>vm.min_free_kbytes</code>, but this would result in an ammount of total free memory less than <code>vm.min_free_kbytes</code> then the OS will clean up memory to service the request.</p>
<p>If <code>vm.min_free_kbytes</code> is less than the ammount requested and there is no virtual memory left, then the system is officially unable to service the request and will lauch the OOMKiller (Out of Memory Killer) to free memory by kiling memory glut processes.</p>
<p>Increase the <code>vm.min_free_kbytes</code> value to prevent this scenario.</p>
<h4 id="the_oom_killer">The OOM Killer<a class="headerlink" href="#the_oom_killer" title="Permanent link">&para;</a></h4>
<p>The OOM kill decides which program to kill to reclaim memory, since hydrus loves memory it is usually picked first, even if another program asking for memory caused the OOM condition. Setting the minimum free kilobytes higher will avoid the running of the OOMkiller which is always preferable, and almost always preventable.</p>
<h4 id="memory_overcommmit">Memory Overcommmit<a class="headerlink" href="#memory_overcommmit" title="Permanent link">&para;</a></h4>
<p>We mentioned that Linux will keep giving out memory, but actually it's possible for Linux to launch the OOM killer if it just feel like our program is aking for too much memory too quickly. Since hydrus is a heavyweight scientific processing package we need to turn this feature off. To turn it off change the value of <code>vm.overcommit_memory</code> which defaults to <code>2</code>.</p>
<p>Set <code>vm.overcommit_memory=1</code> this prevents the OS from using a heuristic and it will just always give memory to anyone who asks for it.</p>
<h4 id="what_about_swappiness">What about swappiness?<a class="headerlink" href="#what_about_swappiness" title="Permanent link">&para;</a></h4>
<p>Swapiness is a setting you might have seen, but it only determines Linux's desire to spend a little bit of time moving memory you haven't touched in a while out of real memory and into virtual memory, it will not prevent the OOM condition it just determines how much time to use for moving things into swap.</p>
<h2 id="why_does_my_linux_system_studder_or_become_unresponsive_when_hydrus_has_been_running_a_while">Why does my Linux system studder or become unresponsive when hydrus has been running a while?<a class="headerlink" href="#why_does_my_linux_system_studder_or_become_unresponsive_when_hydrus_has_been_running_a_while" title="Permanent link">&para;</a></h2>
<p>You are running out of pages because Linux releases I/O buffer pages only when a file is closed, OR memory fragmentation in Hydrus is high because you have a big session weight or had a big I/O spike. Thus the OS is waiting for you to hit the watermark(as described in "why is hydrus crashing") to start freeing pages, which causes the chug.</p>
<p>When contents is written from memory to disk the page is retained so that if you reread that part of the disk the OS does not need to access disk it just pulls it from the much faster memory. This is usually a good thing, but Hydrus makes many small writes to files you probably wont be asking for again soon it eats up pages over time. </p>
<p>Hydrus also holds the database open and red/wrires new areas to it often even if it will not acess those parts again for ages. It tends to accumulate lots of I/O cache for these small pages it will not be interested in. This is really good for hydrus (because it will over time have the root of the most important indexes in memory) but sucks for the responsiveness of other apps, and will cause hydrus to consume pages after doing a lengthy operation in anticipation of needing them again, even when it is thereafter idle. You need to set <code>vm.dirtytime_expire_seconds</code> to a lower value.</p>
<blockquote>
<p><code>vm.dirtytime_expire_seconds</code>
When a lazytime inode is constantly having its pages dirtied, the inode with
an updated timestamp will never get chance to be written out. And, if the
only thing that has happened on the file system is a dirtytime inode caused
by an atime update, a worker will be scheduled to make sure that inode
eventually gets pushed out to disk. This tunable is used to define when dirty
inode is old enough to be eligible for writeback by the kernel flusher threads.
And, it is also used as the interval to wakeup dirtytime writeback thread.</p>
</blockquote>
<p>On many distros this happens only once every 12 hours, try setting it close to every one hour or 2. This will cause the OS to drop pages that were written over 1-2 hours ago. Returning them to the free store for use by other programs.</p>
<p>https://www.kernel.org/doc/Documentation/sysctl/vm.txt</p>
<h2 id="why_does_everything_become_clunky_for_a_bit_if_i_have_tuned_all_of_the_above_settings_especially_if_i_try_to_do_something_on_the_system_that_isnt_hydrus">Why does everything become clunky for a bit if I have tuned all of the above settings? (especially if I try to do something on the system that isn't hydrus)<a class="headerlink" href="#why_does_everything_become_clunky_for_a_bit_if_i_have_tuned_all_of_the_above_settings_especially_if_i_try_to_do_something_on_the_system_that_isnt_hydrus" title="Permanent link">&para;</a></h2>
<p>The kernel launches a process called <code>kswapd</code> to swap and reclaim memory pages, after hydrus has used pages they need to be returned to the OS (unless fragmentation is preventing this). The OS needs to scan for pages allocated to programs which are not in use, it doens't do this all the time because holding the required locks would have a serious performance impact. The behaviour of <code>kswapd</code> is goverened by several important values. If you are using a classic system with a reasonably sized amount of memoery and a swapfile you should tune these. If you are using memory compression (or should be using memory compression because you have a cheap system) read this whole document for info specific to that configuration.</p>
<ul>
<li>
<p><code>vm.watermark_scale_factor</code>
This factor controls the aggressiveness of kswapd. It defines the amount of memory left in a node/system before kswapd is woken up and how much memory needs to be free before kswapd goes back to sleep. The unit is in fractions of 10,000. The default value of 10 means the distances between watermarks are 0.1% of the available memory in the node/system. The maximum value is 1000, or 10% of memory. A high rate of threads entering direct reclaim (allocstall) or kswapd going to sleep prematurely (kswapd_low_wmark_hit_quickly) can indicate that the number of free pages kswapd maintains for latency reasons is too small for the allocation bursts occurring in the system. This knob can then be used to tune kswapd aggressiveness accordingly.</p>
</li>
<li>
<p><code>vm.watermark_boost_factor</code>: If memory fragmentation is high raise the scale factor to look for reclaimable/swappable pages more agressively.</p>
</li>
</ul>
<p>I like to keep <code>watermark_scale_factor</code> at 70 (70/10,000)=0.7%, so kswapd will run until at least 0.7% of system memory has been reclaimed.
i.e. If 32GiB (real and virt) of memory, it will try to keep at least 0.224 GiB immediately available.</p>
<ul>
<li><code>vm.dirty_ratio</code>: The absolute maximum number of un-synced memory(as a percentage of available memory) that the system will buffer before blocking writing processes. This <strong>protects you against OOM, but does not keep your system responsive</strong>.</li>
<li>
<p><strong>Note</strong>: A default installation of Ubuntu sets this way too high (60%) as it does not expect your workload to just be hammering possibly slow disks with written pages. <strong>Even with memory overcomitting this can make you OOM</strong>, because you will run out of real memory before the system pauses the program that is writing so hard. A more reasonable value is 10 (10%)</p>
</li>
<li>
<p><code>vm.dirty_background_ratio</code>: How many the number of unsynced pages that can exist before the system starts comitting them in the background. If this is set too low the system will constantly spend cycles trying to write out dirty pages. If it is set too high it will be way to lazy. I like to set it to 8.</p>
</li>
<li>
<p><code>vm.vfs_cache_pressure</code> The tendancy for the kernel to reclaim I/O cache for files and directories. This is less important than the other values, but hydrus opens and closes lots of file handles so you may want to boost it a bit higher than default. Default=100, set to 110 to bias the kernel into reclaiming I/O pages over keeping them at a "fair rate" compared to other pages. Hydrus tends to write a lot of files and then ignore them for a long time, so its a good idea to prefer freeing pages for infrequent I/O.
<strong>Note</strong>: Increasing <code>vfs_cache_pressure</code> significantly beyond 100 may have negative performance impact. Reclaim code needs to take various locks to find freeable directory and inode objects. With <code>vfs_cache_pressure=1000</code>, it will look for ten times more freeable objects than there are.</p>
</li>
</ul>
<h3 id="virtual_memory_under_linux_4_unleash_the_memory">Virtual Memory Under Linux 4: Unleash the memory<a class="headerlink" href="#virtual_memory_under_linux_4_unleash_the_memory" title="Permanent link">&para;</a></h3>
<p>An example <code>/etc/sysctl.conf</code> section for virtual memory settings.</p>
<div class="highlight"><pre><span></span><code><span class="c1">########</span>
<span class="c1"># virtual memory</span>
<span class="c1">########</span>
<span class="c1">#1 always overcommit, prevents the kernel from using a heuristic to decide that a process is bad for asking for a lot of memory at once and killing it.</span>
<span class="c1">#https://www.kernel.org/doc/Documentation/vm/overcommit-accounting</span>
<span class="na">vm.overcommit_memory</span><span class="o">=</span><span class="s">1</span>
<span class="c1">#force linux to reclaim pages if under a gigabyte </span>
<span class="c1">#is available so large chunk allocates don&#39;t fire off the OOM killer</span>
<span class="na">vm.min_free_kbytes</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">1153434</span>
<span class="c1">#Start freeing up pages that have been written but which are in open files, after 2 hours.</span>
<span class="c1">#Allows pages in long lived files to be reclaimed</span>
<span class="na">vm.dirtytime_expire_seconds</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">7200</span>
<span class="c1">#Have kswapd try to reclaim .7% = 70/10000 of pages before returning to sleep</span>
<span class="c1">#This increases responsiveness by reclaiming a larger portion of pages in low memory condition</span>
<span class="c1">#So that the next time you make a large allocation the kernel doesn&#39;t have to stall and look for pages to free immediately.</span>
<span class="na">vm.watermark_scale_factor</span><span class="o">=</span><span class="s">70</span>
<span class="c1">#Have the kernel prefer to reclaim I/O pages at 110% of the rate at which it frees other pages.</span>
<span class="c1">#Don&#39;t set this value much over 100 or the kernel will spend all its time reclaiming I/O pages</span>
<span class="na">vm.vfs_cache_pressure</span><span class="o">=</span><span class="s">110</span>
</code></pre></div>
<h2 id="virtual_memory_under_linux_5_phenomenal_cosmic_power_itty_bitty_living_space">Virtual Memory Under Linux 5: Phenomenal Cosmic Power; Itty bitty living space<a class="headerlink" href="#virtual_memory_under_linux_5_phenomenal_cosmic_power_itty_bitty_living_space" title="Permanent link">&para;</a></h2>
<p>Are you trying to <strong>run hydrus on a 200 dollar miniPC</strong>, this is suprisingly doable, but you will need to really understand what you are tuning.</p>
<p>To start lets explain memory tiers. As memory moves further away from the CPU it becomes slower. Memory close to the CPU is volatile, which means if you remove power from it, it disappears forever. Conversely disk is called non-volatile memory, or persistant storage. We want to get files written to non-volatile storage, and we don't want to have to compete to read non-volatile storage, we would also prefer to not have to compete for writing, butthis is harder.</p>
<p>The most straight forward way of doing this is to seperate where hydrus writes its SQLITE database(index) files, from where it writes the imported files. But we can make a more flexible setup that will also keep our system responsive, we just need to make sure that the system writes to the fastest possible place first. So let's illustrate the options.</p>
<pre class="mermaid"><code> graph
direction LR
CPU--&gt;RAM;
RAM--&gt;ZRAM;
ZRAM--&gt;SSD;
SSD--&gt;HDD;
subgraph Non-Volatile
SSD;
HDD;
end</code></pre>
<ol>
<li><strong>RAM</strong>: Information must be in RAM for it to be operated on</li>
<li><strong>ZRAM</strong>: A compressed area of RAM that cannot be directly accessed. Like a zip file but in memory. Or for the more technical, like a compressed ramdisk.</li>
<li><strong>SSD</strong>: Fast non-volatile storage,good for random access about 100-1000x slower than RAM.</li>
<li><strong>HDD</strong>: Slow non-volatile storage good for random access. About 10000x slowee than RAM.</li>
<li><strong>Tape</strong>(Not shown): Slow archival storage or backup. surprisingly fast actually but can only be accessed sequentially.</li>
</ol>
<p>The objective is to make the most of our limited hardware so we definitely want to go through zram first. Depending on your configuration you might have a bulk storage (NAS) downstream that you can write the files to, if all of your storage is in the same tower as you are running hydrus, then make sure the SQLITE .db files are on an SSD volume.</p>
<p>Next you should enable <a href="https://wiki.archlinux.org/title/Zram">ZRAM</a> devices (<strong><strong>Not to be confused with ZWAP</strong></strong>). A ZRAM device is a compressed swapfile that lives in RAM.</p>
<p>ZRAM can drastically improve performance, and RAM capacity. Experimentally, a 1.7GB partition usually shrinks to around 740MiB. Depending on your system ZRAM may generate several partitions. The author asked for 4x2GB=8GIB partitions, hence the cited ratio.</p>
<p><strong>ZRAM must be created every boot as RAM-disks are lost when power is removed</strong>
Install a zram generator as part of your startup process. If you still do not have enough swap, you can still create a swapfile. RAM can be configured to use a partition as fallback, but not a file. However you can enable a standard swapfile as described in the prior section. ZRAM generators usually create ZRAM partitions with the highest priority (lowest priority number) so ZRAM will fill up first, before normal disk swaping.</p>
<p>To check your swap configuration
<div class="highlight"><pre><span></span><code>swapon<span class="w"> </span><span class="c1">#no argument</span>
cat<span class="w"> </span>/proc/swapinfo
</code></pre></div></p>
<p>To make maximum use of your swap make sure to <strong><strong>SET THE FOLLOWING VM SETTINGS</strong></strong></p>
<div class="highlight"><pre><span></span><code><span class="c1">#disable IO clustering we are writing to memroy which is super fast</span>
<span class="c1">#IF YOU DO NOT DO THIS YOUR SYSTEM WILL HITCH as it tries to lock multiple RAM pages. This would be desirable on non-volatile storages but is actually bad on RAM.</span>
vm.page-cluster<span class="o">=</span><span class="m">0</span>
<span class="c1">#Tell the system that it costs almost as much to swap as to write out dirty pages. But bias it very slightly to completing writes. This is ideal since hydrus tends to hammer the system with writes, and we want to use ZRAM to eat spikes, but also want the system to slightly prefer writing dirty pages</span>
vm.swappiness<span class="o">=</span><span class="m">99</span>
</code></pre></div>
<hr />
<p>The above is good for most users. If however you also need to speed up your storage due to a high number of applications on your network using it you may wish to install cache, provided you have at least one or two avialable SSD slots, and the writing pattern is many small random writes.</p>
<p>You should never create a write cache without knowing what you are doing. You need two SSDs to crosscheck eachother, and ideally ressilant server SSDs with large capacitors that ensure all content is always written. If you go with a commercial storage solution they will probably check this already, and give you a nice interface for just inserting and assigning SSD cache.</p>
<p>You can also create a cach manually wit the <strong>Logical Volume</strong> using the LVM. If you do this you can group together storage volumes. In particular you can put a <a href="https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/logical_volume_manager_administration/lvm_cache_volume_creation">read or write cache</a> with an SSD in front of slower HDD. </p>
<aside class="md-source-file">
<span class="md-source-file__fact">
<span class="md-icon" title="Last update">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg>
</span>
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">February 28, 2024</span>
</span>
</aside>
</article>
</div>
<script>var tabs=__md_get("__tabs");if(Array.isArray(tabs))e:for(var set of document.querySelectorAll(".tabbed-set")){var tab,labels=set.querySelector(".tabbed-labels");for(tab of tabs)for(var label of labels.getElementsByTagName("label"))if(label.innerText.trim()===tab){var input=document.getElementById(label.htmlFor);input.checked=!0;continue e}}</script>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12Z"/></svg>
Back to top
</button>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
<div class="md-social">
<a href="https://github.com/hydrusnetwork/hydrus" target="_blank" rel="noopener" title="github.com" class="md-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
</a>
<a href="https://twitter.com/hydrusnetwork" target="_blank" rel="noopener" title="twitter.com" class="md-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M459.37 151.716c.325 4.548.325 9.097.325 13.645 0 138.72-105.583 298.558-298.558 298.558-59.452 0-114.68-17.219-161.137-47.106 8.447.974 16.568 1.299 25.34 1.299 49.055 0 94.213-16.568 130.274-44.832-46.132-.975-84.792-31.188-98.112-72.772 6.498.974 12.995 1.624 19.818 1.624 9.421 0 18.843-1.3 27.614-3.573-48.081-9.747-84.143-51.98-84.143-102.985v-1.299c13.969 7.797 30.214 12.67 47.431 13.319-28.264-18.843-46.781-51.005-46.781-87.391 0-19.492 5.197-37.36 14.294-52.954 51.655 63.675 129.3 105.258 216.365 109.807-1.624-7.797-2.599-15.918-2.599-24.04 0-57.828 46.782-104.934 104.934-104.934 30.213 0 57.502 12.67 76.67 33.137 23.715-4.548 46.456-13.32 66.599-25.34-7.798 24.366-24.366 44.833-46.132 57.827 21.117-2.273 41.584-8.122 60.426-16.243-14.292 20.791-32.161 39.308-52.628 54.253z"/></svg>
</a>
<a href="https://hydrus.tumblr.com/" target="_blank" rel="noopener" title="hydrus.tumblr.com" class="md-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 320 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M309.8 480.3c-13.6 14.5-50 31.7-97.4 31.7-120.8 0-147-88.8-147-140.6v-144H17.9c-5.5 0-10-4.5-10-10v-68c0-7.2 4.5-13.6 11.3-16 62-21.8 81.5-76 84.3-117.1.8-11 6.5-16.3 16.1-16.3h70.9c5.5 0 10 4.5 10 10v115.2h83c5.5 0 10 4.4 10 9.9v81.7c0 5.5-4.5 10-10 10h-83.4V360c0 34.2 23.7 53.6 68 35.8 4.8-1.9 9-3.2 12.7-2.2 3.5.9 5.8 3.4 7.4 7.9l22 64.3c1.8 5 3.3 10.6-.4 14.5z"/></svg>
</a>
<a href="mailto:hydrus.admin@gmail.com" target="_blank" rel="noopener" title="" class="md-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M48 64C21.5 64 0 85.5 0 112c0 15.1 7.1 29.3 19.2 38.4l217.6 163.2c11.4 8.5 27 8.5 38.4 0l217.6-163.2c12.1-9.1 19.2-23.3 19.2-38.4 0-26.5-21.5-48-48-48H48zM0 176v208c0 35.3 28.7 64 64 64h384c35.3 0 64-28.7 64-64V176L294.4 339.2a63.9 63.9 0 0 1-76.8 0L0 176z"/></svg>
</a>
<a href="https://discord.gg/wPHPCUZ" target="_blank" rel="noopener" title="discord.gg" class="md-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M524.531 69.836a1.5 1.5 0 0 0-.764-.7A485.065 485.065 0 0 0 404.081 32.03a1.816 1.816 0 0 0-1.923.91 337.461 337.461 0 0 0-14.9 30.6 447.848 447.848 0 0 0-134.426 0 309.541 309.541 0 0 0-15.135-30.6 1.89 1.89 0 0 0-1.924-.91 483.689 483.689 0 0 0-119.688 37.107 1.712 1.712 0 0 0-.788.676C39.068 183.651 18.186 294.69 28.43 404.354a2.016 2.016 0 0 0 .765 1.375 487.666 487.666 0 0 0 146.825 74.189 1.9 1.9 0 0 0 2.063-.676A348.2 348.2 0 0 0 208.12 430.4a1.86 1.86 0 0 0-1.019-2.588 321.173 321.173 0 0 1-45.868-21.853 1.885 1.885 0 0 1-.185-3.126 251.047 251.047 0 0 0 9.109-7.137 1.819 1.819 0 0 1 1.9-.256c96.229 43.917 200.41 43.917 295.5 0a1.812 1.812 0 0 1 1.924.233 234.533 234.533 0 0 0 9.132 7.16 1.884 1.884 0 0 1-.162 3.126 301.407 301.407 0 0 1-45.89 21.83 1.875 1.875 0 0 0-1 2.611 391.055 391.055 0 0 0 30.014 48.815 1.864 1.864 0 0 0 2.063.7A486.048 486.048 0 0 0 610.7 405.729a1.882 1.882 0 0 0 .765-1.352c12.264-126.783-20.532-236.912-86.934-334.541ZM222.491 337.58c-28.972 0-52.844-26.587-52.844-59.239s23.409-59.241 52.844-59.241c29.665 0 53.306 26.82 52.843 59.239 0 32.654-23.41 59.241-52.843 59.241Zm195.38 0c-28.971 0-52.843-26.587-52.843-59.239s23.409-59.241 52.843-59.241c29.667 0 53.307 26.82 52.844 59.239 0 32.654-23.177 59.241-52.844 59.241Z"/></svg>
</a>
<a href="https://www.patreon.com/hydrus_dev" target="_blank" rel="noopener" title="www.patreon.com" class="md-social__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.5.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M489.7 153.8c-.1-65.4-51-119-110.7-138.3C304.8-8.5 207-5 136.1 28.4 50.3 68.9 23.3 157.7 22.3 246.2 21.5 319 28.7 510.6 136.9 512c80.3 1 92.3-102.5 129.5-152.3 26.4-35.5 60.5-45.5 102.4-55.9 72-17.8 121.1-74.7 121-150z"/></svg>
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"base": ".", "features": ["navigation.tracking", "navigation.sections", "navigation.tabs", "content.tabs.link", "navigation.top", "search.suggest", "content.code.annotate", "navigation.instant", "content.action.edit"], "search": "assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
<script src="assets/javascripts/bundle.bd41221c.min.js"></script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink